network: fix behavior-derived WinHTTP requests missing domain/port

lib/cuckoo/common/network_utils.py

@@ -68,6 +68,8 @@
 
 _HEX_HANDLE_RE = re.compile(r"^(?:0x)?([0-9a-fA-F]+)$")
 
+WINHTTP_FLAG_SECURE = 0x00800000
+
 
 def _norm_domain(d):
     if not d or not isinstance(d, str):
@@ -302,7 +304,7 @@ def _parse_handle(v):
     if isinstance(v, int):
         if v <= 0:
             return None
-        return "0x%x" % v
+        return f"0x{v:x}"
     with suppress(Exception):
         s = str(v).strip()
         if not s:
@@ -313,7 +315,7 @@ def _parse_handle(v):
         n = int(m.group(1), 16)
         if n <= 0:
             return None
-        return "0x%x" % n
+        return f"0x{n:x}"
     return None
 
 
@@ -454,7 +456,7 @@ def winhttp_update_from_call(pstate, api_lc, args_map, ret_handle):
 
                 if conn.get("server") and req.get("object"):
                     scheme = "https" if conn.get("port") == 443 else "http"
-                    req["url"] = "%s://%s%s" % (scheme, conn["server"], req["object"])
+                    req["url"] = f"{scheme}://{conn['server']}{req['object']}"
         return
 
     # WinHttpSetOption -> applies to session/connect/request by handle
@@ -484,68 +486,71 @@ def winhttp_finalize_sessions(state):
     procs = (state or {}).get("processes") or {}
 
     for _, p in procs.items():
-        sessions = (p.get("sessions") or {})
-        if not sessions:
+        sessions = p.get("sessions") or {}
+        connects = p.get("connects") or {}
+        if not connects:
             continue
 
         sessions_by_domain = {}
         sessions_by_domain_keys = defaultdict(set)
 
-        for s in sessions.values():
+        # Walk connects directly: WinHttpOpen may be missing (hooked late / failed),
+        # which previously orphaned the connect and dropped all its requests.
+        for c in connects.values():
+            if not isinstance(c, dict):
+                continue
+
+            s = sessions.get(c.get("session_handle")) or {}
             ua = s.get("user_agent") or ""
             access_type = s.get("access_type") or ""
             proxy_name = s.get("proxy_name") or ""
             proxy_bypass = s.get("proxy_bypass") or ""
 
-            for c in s.get("connections") or []:
-                if not isinstance(c, dict):
-                    continue
+            dom = _norm_domain(c.get("server") or "")
+            if not dom:
+                continue
+
+            port = c.get("port")
 
-                server = c.get("server") or ""
-                dom = _norm_domain(server)
-                if not dom:
+            for r in c.get("requests") or []:
+                if not isinstance(r, dict):
                     continue
 
-                port = c.get("port")
-                scheme = "https" if port == 443 else "http"
-
-                for r in c.get("requests") or []:
-                    if not isinstance(r, dict):
-                        continue
-
-                    obj = r.get("object") or ""
-                    if not isinstance(obj, str):
-                        obj = str(obj)
-
-                    obj = obj.strip()
-                    if not obj:
-                        continue
-
-                    if not obj.startswith("/"):
-                        obj = "/" + obj
-
-                    verb = r.get("verb") or ""
-                    if not isinstance(verb, str):
-                        verb = str(verb)
-
-                    verb = verb.strip().upper() or "GET"
-                    request = f"{verb} {obj} \r\nUser-Agent: {ua}\r\nHost: {dom}\r\n"
-                    entry = {
-                        "uri": obj,
-                        "dport": port,
-                        "method": verb,
-                        "protocol": scheme,
-                        "user_agent": ua,
-                        "request": request,
-                        "access_type": access_type,
-                        "proxy_name": proxy_name,
-                        "proxy_bypass": proxy_bypass,
-                    }
-
-                    key = (obj, verb, ua, access_type, proxy_name, proxy_bypass)
-                    if key not in sessions_by_domain_keys[dom]:
-                        sessions_by_domain.setdefault(dom, []).append(entry)
-                        sessions_by_domain_keys[dom].add(key)
+                obj = str(r.get("object") or "").strip()
+                if not obj:
+                    continue
+                if not obj.startswith("/"):
+                    obj = "/" + obj
+
+                verb = str(r.get("verb") or "").strip().upper() or "GET"
+
+                flags = _safe_int(r.get("flags")) or 0
+                secure = bool(flags & WINHTTP_FLAG_SECURE) or port == 443
+                scheme = "https" if secure else "http"
+                default_port = 443 if secure else 80
+                # port 0 == INTERNET_DEFAULT_PORT
+                dport = default_port if port in (None, 0) else port
+                netloc = dom if dport == default_port else f"{dom}:{dport}"
+                url = f"{scheme}://{netloc}{obj}"
+
+                request = f"{verb} {obj} \r\nUser-Agent: {ua}\r\nHost: {netloc}\r\n"
+                entry = {
+                    "url": url,
+                    "uri": obj,
+                    "dport": dport,
+                    "method": verb,
+                    "protocol": scheme,
+                    "user_agent": ua,
+                    "request": request,
+                    "access_type": access_type,
+                    "proxy_name": proxy_name,
+                    "proxy_bypass": proxy_bypass,
+                }
+
+                key = (url, verb, ua, access_type, proxy_name, proxy_bypass)
+                if key not in sessions_by_domain_keys[dom]:
+                    sessions_by_domain.setdefault(dom, []).append(entry)
+                    sessions_by_domain_keys[dom].add(key)
 
         if sessions_by_domain:
             sessions_list = [{"host": dom, "events": evts} for dom, evts in sessions_by_domain.items()]

modules/processing/network.py

@@ -22,7 +22,7 @@
 from hashlib import md5, sha1, sha256
 from itertools import islice
 from json import loads
-from typing import Any, Dict, List, Optional
+from typing import Any
 from urllib.parse import urlparse, urlunparse
 
 import cachetools.func
@@ -146,7 +146,7 @@
             ip_passlist.add(ip)
 
 if enabled_network_passlist and network_passlist_file and os.path.isfile(network_passlist_file):
-    with open(os.path.join(CUCKOO_ROOT, network_passlist_file), "r") as f:
+    with open(os.path.join(CUCKOO_ROOT, network_passlist_file)) as f:
         for cidr in set(f.read().splitlines()):
             if cidr.startswith("#") or len(cidr.strip()) == 0:
                 # comment or empty line
@@ -496,13 +496,13 @@ def _add_dns(self, udpdata, ts):
                     ans = {"type": "A"}
                     try:
                         ans["data"] = socket.inet_ntoa(answer.rdata)
-                    except socket.error:
+                    except OSError:
                         continue
                 elif answer.type == dpkt.dns.DNS_AAAA:
                     ans = {"type": "AAAA"}
                     try:
                         ans["data"] = socket.inet_ntop(socket.AF_INET6, answer.rdata)
-                    except (socket.error, ValueError):
+                    except (OSError, ValueError):
                         continue
                 elif answer.type == dpkt.dns.DNS_CNAME:
                     ans = {"type": "CNAME", "data": answer.cname}
@@ -767,7 +767,7 @@ def run(self):
 
         try:
             file = open(self.filepath, "rb")
-        except (IOError, OSError):
+        except OSError:
             log.error("Unable to open %s", self.filepath)
             return self.results
 
@@ -1129,7 +1129,7 @@ def _import_ja3_fprints(self):
         """
         ja3_fprints = {}
         if path_exists(self.ja3_file):
-            with open(self.ja3_file, "r") as fpfile:
+            with open(self.ja3_file) as fpfile:
                 for line in fpfile:
                     try:
                         ja3 = loads(line)
@@ -1140,7 +1140,7 @@ def _import_ja3_fprints(self):
 
         return ja3_fprints
 
-    def _load_network_map(self) -> Dict:
+    def _load_network_map(self) -> dict:
         with suppress(Exception):
             behavior_net_map = self.results.get("behavior", {}).get("network_map") or {}
             if not behavior_net_map:
@@ -1171,7 +1171,7 @@ def _load_network_map(self) -> Dict:
             return net_map
         return {}
 
-    def _reconstruct_endpoint_map(self, raw_map) -> Dict[tuple, List[Dict]]:
+    def _reconstruct_endpoint_map(self, raw_map) -> dict[tuple, list[dict]]:
         """
         Convert JSON-friendly "ip:port" keys back to (ip, int(port)) tuples.
         """
@@ -1193,7 +1193,7 @@ def _reconstruct_endpoint_map(self, raw_map) -> Dict[tuple, List[Dict]]:
                     continue
         return endpoint_map
 
-    def _pick_best(self, candidates: List[Dict]) -> Optional[Dict]:
+    def _pick_best(self, candidates: list[dict]) -> dict | None:
         if not candidates:
             return None
 
@@ -1203,7 +1203,7 @@ def _pick_best(self, candidates: List[Dict]) -> Optional[Dict]:
 
         return candidates[0]
 
-    def _match_dns_process(self, dns_entry: Dict, dns_intents: Dict, max_skew_seconds: float = 10.0) -> Optional[Dict]:
+    def _match_dns_process(self, dns_entry: dict, dns_intents: dict, max_skew_seconds: float = 10.0) -> dict | None:
         """
         Match a network.dns entry to the closest behavior DNS intent by:
           - same domain
@@ -1241,7 +1241,7 @@ def _match_dns_process(self, dns_entry: Dict, dns_intents: Dict, max_skew_second
 
         return candidates[0].get("process")
 
-    def _pcap_first_epoch(self, network: Dict) -> Optional[float]:
+    def _pcap_first_epoch(self, network: dict) -> float | None:
         ts = []
         for k in ("dns", "http"):
             for e in network.get(k) or []:
@@ -1250,7 +1250,7 @@ def _pcap_first_epoch(self, network: Dict) -> Optional[float]:
                     ts.append(float(v))
         return min(ts) if ts else None
 
-    def _build_dns_events_rel(self, network: Dict, dns_intents: Dict, max_skew_seconds: float = 10.0) -> List[Dict]:
+    def _build_dns_events_rel(self, network: dict, dns_intents: dict, max_skew_seconds: float = 10.0) -> list[dict]:
         """
         Returns a list of dns events:
         [{"t_rel": float, "process": {...}|None, "request": "example.com"}]
@@ -1271,7 +1271,7 @@ def _build_dns_events_rel(self, network: Dict, dns_intents: Dict, max_skew_secon
         out.sort(key=lambda x: x["t_rel"])
         return out
 
-    def _nearest_dns_process_by_rel_time(self, dns_events_rel: List[Dict], t_rel: Any, max_skew: float = 5.0) -> Optional[Dict]:
+    def _nearest_dns_process_by_rel_time(self, dns_events_rel: list[dict], t_rel: Any, max_skew: float = 5.0) -> dict | None:
         if not dns_events_rel or not isinstance(t_rel, (int, float)):
             return None
 
@@ -1287,7 +1287,7 @@ def _nearest_dns_process_by_rel_time(self, dns_events_rel: List[Dict], t_rel: An
             return best.get("process")
         return None
 
-    def _set_proc_fields(self, obj: Dict, proc: Optional[Dict]):
+    def _set_proc_fields(self, obj: dict, proc: dict | None):
         """
         Add process_id/process_name onto an existing network entry.
         If proc is None, sets them to None (keeps template stable).
@@ -1299,7 +1299,7 @@ def _set_proc_fields(self, obj: Dict, proc: Optional[Dict]):
             obj["process_id"] = None
             obj["process_name"] = None
 
-    def _process_map(self, network: Dict):
+    def _process_map(self, network: dict):
         net_map = self._load_network_map()
 
         if not network or not net_map:
@@ -1425,29 +1425,37 @@ def _merge_behavior_network(self, network):
                     if not sessions:
                         continue
 
-                    # Use first session entry as representative
-                    s0 = sessions[0] or {}
-                    method = s0.get("method") or ""
-                    dport = s0.get("port")
-                    uri = s0.get("uri") or "/"
-                    protocol = s0.get("protocol")
-
-                    entry = {
-                        "host": hnorm,
-                        "dport": dport,
-                        "uri": uri,
-                        "method": method,
-                        "data": s0.get("request"),
-                        "protocol": protocol,
-                        "access_type": s0.get("access_type"),
-                        "proxy_name": s0.get("proxy_name"),
-                        "proxy_bypass": s0.get("proxy_bypass"),
-                        "source": "behavior",
-                        "process_id": p.get("process_id"),
-                        "process_name": p.get("process_name"),
-                    }
+                    seen_urls = set()
+                    for s in sessions:
+                        s = s or {}
+                        path = s.get("uri") or "/"
+                        protocol = s.get("protocol") or "http"
+                        url = s.get("url") or f"{protocol}://{hnorm}{path}"
+                        if url in seen_urls:
+                            continue
+                        seen_urls.add(url)
+
+                        dport = s.get("dport")
+                        entry = {
+                            "host": hnorm,
+                            "dport": dport,
+                            "port": dport,
+                            # network.http template renders only `uri`; PCAP entries store the full URL there.
+                            "uri": url,
+                            "path": path,
+                            "method": s.get("method") or "",
+                            "data": s.get("request"),
+                            "protocol": protocol,
+                            "user_agent": s.get("user_agent"),
+                            "access_type": s.get("access_type"),
+                            "proxy_name": s.get("proxy_name"),
+                            "proxy_bypass": s.get("proxy_bypass"),
+                            "source": "behavior",
+                            "process_id": p.get("process_id"),
+                            "process_name": p.get("process_name"),
+                        }
+                        network.setdefault("http", []).append(entry)
 
-                    network.setdefault("http", []).append(entry)
                     existing_hosts.add(hnorm)
 
         # DNS
@@ -1652,7 +1660,7 @@ def get_tlsmaster(self):
         if not path_exists(dump_tls_log):
             return tlsmaster
 
-        with open(dump_tls_log, "r") as f:
+        with open(dump_tls_log) as f:
             for entry in f:
                 try:
                     for m in re.finditer(
@@ -1863,8 +1871,7 @@ def packets_for_stream(fobj, offset):
     ts, raw = next(pcapiter)
 
     fobj.seek(offset)
-    for p in next_connection_packets(pcapiter, linktype=pcap.datalink()):
-        yield p
+    yield from next_connection_packets(pcapiter, linktype=pcap.datalink())
 
 
 def check_pcap_file_type(filepath):