1
2
3
4
5 import os
6 import logging
7 import datetime
8
9 from lib.cuckoo.common.abstracts import Processing
10 from lib.cuckoo.common.config import Config
11 from lib.cuckoo.common.netlog import NetlogParser, BsonParser
12 from lib.cuckoo.common.utils import convert_to_printable, logtime
13 from lib.cuckoo.common.utils import cleanup_value
14
15 log = logging.getLogger(__name__)
16
18 """Fix a registry key to have it normalized.
19 @param key: raw key
20 @returns: normalized key
21 """
22 res = key
23 if key.lower().startswith("registry\\machine\\"):
24 res = "HKEY_LOCAL_MACHINE\\" + key[17:]
25 elif key.lower().startswith("registry\\user\\"):
26 res = "HKEY_USERS\\" + key[14:]
27 elif key.lower().startswith("\\registry\\machine\\"):
28 res = "HKEY_LOCAL_MACHINE\\" + key[18:]
29 elif key.lower().startswith("\\registry\\user\\"):
30 res = "HKEY_USERS\\" + key[15:]
31
32 if not res.endswith("\\\\"):
33 res = res + "\\"
34 return res
35
37 """Parses process log file."""
38
40 """@param log_path: log file path."""
41 self._log_path = log_path
42 self.fd = None
43 self.parser = None
44
45 self.process_id = None
46 self.process_name = None
47 self.parent_id = None
48 self.first_seen = None
49 self.calls = self
50 self.lastcall = None
51
52 if os.path.exists(log_path) and os.stat(log_path).st_size > 0:
53 self.parse_first_and_reset()
54
56 self.fd = open(self._log_path, "rb")
57
58 if self._log_path.endswith(".bson"):
59 self.parser = BsonParser(self)
60 elif self._log_path.endswith(".raw"):
61 self.parser = NetlogParser(self)
62 else:
63 self.fd.close()
64 self.fd = None
65 return
66
67
68
69 while not self.process_id:
70 self.parser.read_next_message()
71
72 self.fd.seek(0)
73
74 - def read(self, length):
75 if not length:
76 return ''
77 buf = self.fd.read(length)
78 if not buf or len(buf) != length:
79 raise EOFError()
80 return buf
81
86
88 return "<ParseProcessLog log-path: %r>" % self._log_path
89
92
94 self.fd.seek(0)
95 self.lastcall = None
96
98 """Compare two calls for equality. Same implementation as before netlog.
99 @param a: call a
100 @param b: call b
101 @return: True if a == b else False
102 """
103 if a["api"] == b["api"] and \
104 a["status"] == b["status"] and \
105 a["arguments"] == b["arguments"] and \
106 a["return"] == b["return"]:
107 return True
108 return False
109
111 while not self.lastcall:
112 r = None
113 try:
114 r = self.parser.read_next_message()
115 except EOFError:
116 return False
117
118 if not r:
119 return False
120 return True
121
123 if not self.fd:
124 raise StopIteration()
125
126 if not self.wait_for_lastcall():
127 self.reset()
128 raise StopIteration()
129
130 nextcall, self.lastcall = self.lastcall, None
131
132 self.wait_for_lastcall()
133 while self.lastcall and self.compare_calls(nextcall, self.lastcall):
134 nextcall["repeated"] += 1
135 self.lastcall = None
136 self.wait_for_lastcall()
137
138 return nextcall
139
140 - def log_process(self, context, timestring, pid, ppid, modulepath, procname):
141 self.process_id, self.parent_id, self.process_name = pid, ppid, procname
142 self.first_seen = timestring
143
146
147 - def log_call(self, context, apiname, category, arguments):
148 apiindex, status, returnval, tid, timediff = context
149
150 current_time = self.first_seen + datetime.timedelta(0, 0, timediff*1000)
151 timestring = logtime(current_time)
152
153 self.lastcall = self._parse([timestring,
154 tid,
155 category,
156 apiname,
157 status,
158 returnval] + arguments)
159
161 log.warning("ParseProcessLog error condition on log %s: %s", str(self._log_path), emsg)
162
164 """Parse log row.
165 @param row: row data.
166 @return: parsed information dict.
167 """
168 call = {}
169 arguments = []
170
171 try:
172 timestamp = row[0]
173 thread_id = row[1]
174 category = row[2]
175 api_name = row[3]
176 status_value = row[4]
177 return_value = row[5]
178 except IndexError as e:
179 log.debug("Unable to parse process log row: %s", e)
180 return None
181
182
183
184 for index in range(6, len(row)):
185 argument = {}
186
187
188 try:
189 arg_name, arg_value = row[index]
190 except ValueError as e:
191 log.debug("Unable to parse analysis row argument (row=%s): %s", row[index], e)
192 continue
193
194 argument["name"] = arg_name
195
196 argument["value"] = convert_to_printable(cleanup_value(arg_value))
197 arguments.append(argument)
198
199 call["timestamp"] = timestamp
200 call["thread_id"] = str(thread_id)
201 call["category"] = category
202 call["api"] = api_name
203 call["status"] = bool(int(status_value))
204
205 if isinstance(return_value, int):
206 call["return"] = "0x%.08x" % return_value
207 else:
208 call["return"] = convert_to_printable(cleanup_value(return_value))
209
210 call["arguments"] = arguments
211 call["repeated"] = 0
212
213 return call
214
216 """Processes analyzer."""
217
219 """@param logs_path: logs path."""
220 self._logs_path = logs_path
221 self.cfg = Config()
222
224 """Run analysis.
225 @return: processes infomartion list.
226 """
227 results = []
228
229 if not os.path.exists(self._logs_path):
230 log.error("Analysis results folder does not exist at path \"%s\".",
231 self._logs_path)
232 return results
233
234 if len(os.listdir(self._logs_path)) == 0:
235 log.error("Analysis results folder does not contain any file.")
236 return results
237
238 for file_name in os.listdir(self._logs_path):
239 file_path = os.path.join(self._logs_path, file_name)
240
241 if os.path.isdir(file_path):
242 continue
243
244
245 if os.stat(file_path).st_size > self.cfg.processing.analysis_size_limit:
246 log.warning("Behavioral log {0} too big to be processed, skipped.".format(file_name))
247 continue
248
249
250 current_log = ParseProcessLog(file_path)
251 if current_log.process_id is None:
252 continue
253
254
255
256 results.append({
257 "process_id": current_log.process_id,
258 "process_name": current_log.process_name,
259 "parent_id": current_log.parent_id,
260 "first_seen": logtime(current_log.first_seen),
261 "calls": current_log.calls,
262 })
263
264
265
266 results.sort(key=lambda process: process["first_seen"])
267
268 return results
269
271 """Generates summary information."""
272
273 key = "summary"
274
276 self.keys = []
277 self.mutexes = []
278 self.files = []
279 self.handles = []
280
282 for known_handle in self.handles:
283 if handle != 0 and handle == known_handle["handle"]:
284 return None
285
286 name = ""
287
288 if registry == 0x80000000:
289 name = "HKEY_CLASSES_ROOT\\"
290 elif registry == 0x80000001:
291 name = "HKEY_CURRENT_USER\\"
292 elif registry == 0x80000002:
293 name = "HKEY_LOCAL_MACHINE\\"
294 elif registry == 0x80000003:
295 name = "HKEY_USERS\\"
296 elif registry == 0x80000004:
297 name = "HKEY_PERFORMANCE_DATA\\"
298 elif registry == 0x80000005:
299 name = "HKEY_CURRENT_CONFIG\\"
300 elif registry == 0x80000006:
301 name = "HKEY_DYN_DATA\\"
302 else:
303 for known_handle in self.handles:
304 if registry == known_handle["handle"]:
305 name = known_handle["name"] + "\\"
306
307 key = fix_key(name + subkey)
308 self.handles.append({"handle": handle, "name": key})
309 return key
310
312 """Generate processes list from streamed calls/processes.
313 @return: None.
314 """
315
316 if call["api"].startswith("RegOpenKeyEx") or call["api"].startswith("RegCreateKeyEx"):
317 registry = 0
318 subkey = ""
319 handle = 0
320
321 for argument in call["arguments"]:
322 if argument["name"] == "Registry":
323 registry = int(argument["value"], 16)
324 elif argument["name"] == "SubKey":
325 subkey = argument["value"]
326 elif argument["name"] == "Handle":
327 handle = int(argument["value"], 16)
328
329 name = self._check_registry(registry, subkey, handle)
330 if name and name not in self.keys:
331 self.keys.append(name)
332 elif call["api"].startswith("NtOpenKey"):
333 registry = -1
334 subkey = ""
335 handle = 0
336
337 for argument in call["arguments"]:
338 if argument["name"] == "ObjectAttributes":
339 subkey = argument["value"]
340 elif argument["name"] == "KeyHandle":
341 handle = int(argument["value"], 16)
342
343 name = self._check_registry(registry, subkey, handle)
344 if name and name not in self.keys:
345 self.keys.append(name)
346 elif call["api"].startswith("NtDeleteValueKey"):
347 registry = -1
348 subkey = ""
349 handle = 0
350
351 for argument in call["arguments"]:
352 if argument["name"] == "ValueName":
353 subkey = argument["value"]
354 elif argument["name"] == "KeyHandle":
355 handle = int(argument["value"], 16)
356
357 name = self._check_registry(registry, subkey, handle)
358 if name and name not in self.keys:
359 self.keys.append(name)
360 elif call["api"].startswith("RegCloseKey"):
361 handle = 0
362
363 for argument in call["arguments"]:
364 if argument["name"] == "Handle":
365 handle = int(argument["value"], 16)
366
367 if handle != 0:
368 for a in self.handles:
369 if a["handle"] == handle:
370 try:
371 self.handles.remove(a)
372 except ValueError:
373 pass
374
375 elif call["category"] == "filesystem":
376 for argument in call["arguments"]:
377 if argument["name"] == "FileName":
378 value = argument["value"].strip()
379 if not value:
380 continue
381
382 if value not in self.files:
383 self.files.append(value)
384
385 elif call["category"] == "synchronization":
386 for argument in call["arguments"]:
387 if argument["name"] == "MutexName":
388 value = argument["value"].strip()
389 if not value:
390 continue
391
392 if value not in self.mutexes:
393 self.mutexes.append(value)
394
396 """Get registry keys, mutexes and files.
397 @return: Summary of keys, mutexes and files.
398 """
399 return {"files": self.files, "keys": self.keys, "mutexes": self.mutexes}
400
402 """Generates a more extensive high-level representation than Summary."""
403
404 key = "enhanced"
405
407 """
408 @param details: Also add some (not so relevant) Details to the log
409 """
410 self.currentdir = "C: "
411 self.eid = 0
412 self.details = details
413 self.filehandles = {}
414 self.servicehandles = {}
415 self.keyhandles = {
416 "0x80000000": "HKEY_CLASSES_ROOT\\",
417 "0x80000001": "HKEY_CURRENT_USER\\",
418 "0x80000002": "HKEY_LOCAL_MACHINE\\",
419 "0x80000003": "HKEY_USERS\\",
420 "0x80000004": "HKEY_PERFORMANCE_DATA\\",
421 "0x80000005": "HKEY_CURRENT_CONFIG\\",
422 "0x80000006": "HKEY_DYN_DATA\\"
423 }
424 self.modules = {}
425 self.procedures = {}
426 self.events = []
427
429 """
430 Add a procedure address
431 """
432 self.procedures[base] = "{0}:{1}".format(self._get_loaded_module(mbase), name)
433
435 """
436 Add a loaded module to the internal database
437 """
438 self.modules[base] = name
439
441 """
442 Get the name of a loaded module from the internal db
443 """
444 return self.modules.get(base, "")
445
446
448 """
449 @registry: returned, new handle
450 @handle: handle to base key
451 @subkey: subkey to add
452 """
453 if handle != 0 and handle in self.keyhandles:
454 return self.keyhandles[handle]
455
456 name = ""
457 if registry and registry != "0x00000000" and \
458 registry in self.keyhandles:
459 name = self.keyhandles[registry]
460
461 nkey = name + subkey
462 nkey = fix_key(nkey)
463
464 self.keyhandles[handle] = nkey
465
466 return nkey
467
475
478
480 """ Gets files calls
481 @return: information list
482 """
483 def _load_args(call):
484 """
485 Load arguments from call
486 """
487 res = {}
488 for argument in call["arguments"]:
489 res[argument["name"]] = argument["value"]
490
491 return res
492
493 def _generic_handle_details(self, call, item):
494 """
495 Generic handling of api calls
496 @call: the call dict
497 @item: Generic item to process
498 """
499 event = None
500 if call["api"] in item["apis"]:
501 args = _load_args(call)
502 self.eid += 1
503
504 event = {
505 "event": item["event"],
506 "object": item["object"],
507 "timestamp": call["timestamp"],
508 "eid": self.eid,
509 "data": {}
510 }
511
512 for logname, dataname in item["args"]:
513 event["data"][logname] = args.get(dataname)
514 return event
515
516 def _generic_handle(self, data, call):
517 """Generic handling of api calls."""
518 for item in data:
519 event = _generic_handle_details(self, call, item)
520 if event:
521 return event
522
523 return None
524
525
526 def _add_handle(handles, handle, filename):
527 handles[handle] = filename
528
529 def _remove_handle(handles, handle):
530 if handle in handles:
531 handles.pop(handle)
532
533 def _get_handle(handles, handle):
534 return handles.get(handle)
535
536 def _get_service_action(control_code):
537 """@see: http://msdn.microsoft.com/en-us/library/windows/desktop/ms682108%28v=vs.85%29.aspx"""
538 codes = {1: "stop",
539 2: "pause",
540 3: "continue",
541 4: "info"}
542
543 default = "user" if control_code >= 128 else "notify"
544 return codes.get(control_code, default)
545
546 event = None
547
548 gendat = [
549 {
550 "event": "move",
551 "object": "file",
552 "apis": [
553 "MoveFileWithProgressW",
554 "MoveFileExA",
555 "MoveFileExW"
556 ],
557 "args": [
558 ("from", "ExistingFileName"),
559 ("to", "NewFileName")
560 ]
561 },
562 {
563 "event": "copy",
564 "object": "file",
565 "apis": [
566 "CopyFileA",
567 "CopyFileW",
568 "CopyFileExW",
569 "CopyFileExA"
570 ],
571 "args": [
572 ("from", "ExistingFileName"),
573 ("to", "NewFileName")
574 ]
575 },
576 {
577 "event": "delete",
578 "object": "file",
579 "apis": [
580 "DeleteFileA",
581 "DeleteFileW",
582 "NtDeleteFile"
583 ],
584 "args": [("file", "FileName")]
585 },
586 {
587 "event": "delete",
588 "object": "dir",
589 "apis": [
590 "RemoveDirectoryA",
591 "RemoveDirectoryW"
592 ],
593 "args": [("file", "DirectoryName")]
594 },
595 {
596 "event": "create",
597 "object": "dir",
598 "apis": [
599 "CreateDirectoryW",
600 "CreateDirectoryExW"
601 ],
602 "args": [("file", "DirectoryName")]
603 },
604 {
605 "event": "write",
606 "object": "file",
607 "apis": [
608 "URLDownloadToFileW",
609 "URLDownloadToFileA"
610 ],
611 "args": [("file", "FileName")]
612 },
613 {
614 "event": "execute",
615 "object": "file",
616 "apis": [
617 "CreateProcessAsUserA",
618 "CreateProcessAsUserW",
619 "CreateProcessA",
620 "CreateProcessW",
621 "NtCreateProcess",
622 "NtCreateProcessEx"
623 ],
624 "args": [("file", "FileName")]
625 },
626 {
627 "event": "execute",
628 "object": "file",
629 "apis": [
630 "CreateProcessInternalW",
631 ],
632 "args": [("file", "CommandLine")]
633 },
634 {
635 "event": "execute",
636 "object": "file",
637 "apis": [
638 "ShellExecuteExA",
639 "ShellExecuteExW",
640 ],
641 "args": [("file", "FilePath")]
642 },
643 {
644 "event": "load",
645 "object": "library",
646 "apis": [
647 "LoadLibraryA",
648 "LoadLibraryW",
649 "LoadLibraryExA",
650 "LoadLibraryExW",
651 "LdrLoadDll",
652 "LdrGetDllHandle"
653 ],
654 "args": [
655 ("file", "FileName"),
656 ("pathtofile", "PathToFile"),
657 ("moduleaddress", "BaseAddress")
658 ]
659 },
660 {
661 "event": "findwindow",
662 "object": "windowname",
663 "apis": [
664 "FindWindowA",
665 "FindWindowW",
666 "FindWindowExA",
667 "FindWindowExW"
668 ],
669 "args": [
670 ("classname", "ClassName"),
671 ("windowname", "WindowName")
672 ]
673 },
674 {
675 "event": "read",
676 "object": "file",
677 "apis": [
678 "NtReadFile",
679 "ReadFile"
680 ],
681 "args": []
682 },
683 {
684 "event": "write",
685 "object": "file",
686 "apis": ["NtWriteFile"],
687 "args": []
688 },
689 {
690 "event": "delete",
691 "object": "registry",
692 "apis": [
693 "RegDeleteKeyA",
694 "RegDeleteKeyW"
695 ],
696 "args": []
697 },
698 {
699 "event": "write",
700 "object": "registry",
701 "apis": [
702 "RegSetValueExA",
703 "RegSetValueExW"
704 ],
705 "args": [
706 ("content", "Buffer"),
707 ("object", "object")
708 ]
709 },
710 {
711 "event": "read",
712 "object": "registry",
713 "apis": [
714 "RegQueryValueExA",
715 "RegQueryValueExW",
716 "NtQueryValueKey"
717 ],
718 "args": []
719 },
720 {
721 "event": "delete",
722 "object": "registry",
723 "apis": [
724 "RegDeleteValueA",
725 "RegDeleteValueW",
726 "NtDeleteValueKey"
727 ],
728 "args": []
729 },
730 {
731 "event": "create",
732 "object": "windowshook",
733 "apis": ["SetWindowsHookExA"],
734 "args": [
735 ("id", "HookIdentifier"),
736 ("moduleaddress", "ModuleAddress"),
737 ("procedureaddress", "ProcedureAddress")
738 ]
739 },
740 {
741 "event": "modify",
742 "object": "service",
743 "apis": ["ControlService"],
744 "args": [("controlcode", "ControlCode")]
745 },
746 {
747 "event": "delete",
748 "object": "service",
749 "apis": ["DeleteService"],
750 "args": [],
751 },
752 ]
753
754
755
756
757
758
759
760
761
762
763 event = _generic_handle(self, gendat, call)
764 args = _load_args(call)
765
766 if event:
767 if call["api"] in ["NtReadFile", "ReadFile", "NtWriteFile"]:
768 event["data"]["file"] = _get_handle(self.filehandles, args["FileHandle"])
769
770 elif call["api"] in ["RegDeleteKeyA", "RegDeleteKeyW"]:
771 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("SubKey", ""))
772
773 elif call["api"] in ["RegSetValueExA", "RegSetValueExW"]:
774 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("ValueName", ""))
775
776 elif call["api"] in ["RegQueryValueExA", "RegQueryValueExW", "RegDeleteValueA", "RegDeleteValueW"]:
777 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "UNKNOWN")), args.get("ValueName", ""))
778
779 elif call["api"] in ["NtQueryValueKey", "NtDeleteValueKey"]:
780 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("KeyHandle", "UNKNOWN")), args.get("ValueName", ""))
781
782 elif call["api"] in ["LoadLibraryA", "LoadLibraryW", "LoadLibraryExA", "LoadLibraryExW", "LdrGetDllHandle"] and call["status"]:
783 self._add_loaded_module(args.get("FileName", ""), args.get("ModuleHandle", ""))
784
785 elif call["api"] in ["LdrLoadDll"] and call["status"]:
786 self._add_loaded_module(args.get("FileName", ""), args.get("BaseAddress", ""))
787
788 elif call["api"] in ["LdrGetProcedureAddress"] and call["status"]:
789 self._add_procedure(args.get("ModuleHandle", ""), args.get("FunctionName", ""), args.get("FunctionAddress", ""))
790 event["data"]["module"] = self._get_loaded_module(args.get("ModuleHandle", ""))
791
792 elif call["api"] in ["SetWindowsHookExA"]:
793 event["data"]["module"] = self._get_loaded_module(args.get("ModuleAddress", ""))
794
795 if call["api"] in ["ControlService", "DeleteService"]:
796 event["data"]["service"] = _get_handle(self.servicehandles, args["ServiceHandle"])
797
798 if call["api"] in ["ControlService"]:
799 event["data"]["action"] = _get_service_action(args["ControlCode"])
800
801 return event
802
803 elif call["api"] in ["SetCurrentDirectoryA", "SetCurrentDirectoryW"]:
804 self.currentdir = args["Path"]
805
806
807 elif call["api"] in ["NtCreateFile", "NtOpenFile"]:
808 _add_handle(self.filehandles, args["FileHandle"], args["FileName"])
809
810 elif call["api"] in ["CreateFileW"]:
811 _add_handle(self.filehandles, call["return"], args["FileName"])
812
813 elif call["api"] in ["NtClose", "CloseHandle"]:
814 _remove_handle(self.filehandles, args["Handle"])
815
816
817 elif call["api"] in ["OpenServiceW"]:
818 _add_handle(self.servicehandles, call["return"], args["ServiceName"])
819
820
821 elif call["api"] in ["RegOpenKeyExA", "RegOpenKeyExW", "RegCreateKeyExA", "RegCreateKeyExW"]:
822 self._add_keyhandle(args.get("Registry", ""), args.get("SubKey", ""), args.get("Handle", ""))
823
824 elif call["api"] in ["NtOpenKey"]:
825 self._add_keyhandle(None, args.get("ObjectAttributes", ""), args.get("KeyHandle", ""))
826
827 elif call["api"] in ["RegCloseKey"]:
828 self._remove_keyhandle(args.get("Handle", ""))
829
830 return event
831
833 """Generate processes list from streamed calls/processes.
834 @return: None.
835 """
836 event = self._process_call(call)
837 if event:
838 self.events.append(event)
839
841 """Get registry keys, mutexes and files.
842 @return: Summary of keys, mutexes and files.
843 """
844 return self.events
845
847 """Generates process tree."""
848
849 key = "processtree"
850
852 self.processes = []
853 self.tree = []
854
856 """Add a node to a process tree.
857 @param node: node to add.
858 @param tree: processes tree.
859 @return: boolean with operation success status.
860 """
861
862 for process in tree:
863
864
865 if process["pid"] == node["parent_id"]:
866 process["children"].append(node)
867
868 else:
869 self.add_node(node, process["children"])
870
872 for entry in self.processes:
873 if entry["pid"] == process["process_id"]:
874 return
875
876 self.processes.append(dict(
877 name=process["process_name"],
878 pid=process["process_id"],
879 parent_id=process["parent_id"],
880 children=[]
881 ))
882
884 children = []
885
886
887 for process in self.processes:
888 has_parent = False
889
890 for process_again in self.processes:
891
892
893 if process_again["pid"] == process["parent_id"]:
894 has_parent = True
895
896
897 if has_parent:
898 children.append(process)
899
900 else:
901 self.tree.append(process)
902
903
904 for process in children:
905 self.add_node(process, self.tree)
906
907 return self.tree
908
910 """Behavior Analyzer."""
911
912 key = "behavior"
913
947