clickgrab.py

#!/usr/bin/env python3
"""
ClickGrab
=========
URL Analyzer & Threat-Intel Collector for Fake CAPTCHA ("ClickFix") campaigns.

This command-line tool fetches suspect URLs (live feeds or user-supplied),
downloads their HTML, and statically analyses the content for indicators of
compromise (PowerShell, OAuth redirection abuse, clipboard hijacking, fake
CAPTCHAs, etc.).  It leverages a pattern library centralised in
`models.CommonPatterns` and modern Pydantic v2 models to return strongly-typed
results that can be rendered as HTML dashboards or ingested as JSON/CSV.

Key Features
------------
• Pull recent feeds from **URLhaus** & **AlienVault OTX** (tag-filtered).
• Detect and decode Base64, obfuscated JavaScript, encoded/hidden PowerShell.
• Extract URLs, IPs, clipboard commands, OAuth flows, and more.
• Risk-score sites and commands; generate HTML/JSON/CSV reports.
• Designed for automation — GitHub Actions workflow provided.

Author : Michael Haag  <https://github.com/MHaggis/ClickGrab>
License: Apache-2.0
"""

import argparse
import csv
import hashlib
import json
import logging
import os
import pathlib
import sys
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional, Tuple
from urllib.parse import urlparse

import requests
from dotenv import load_dotenv

import extractors
from models import (
    ClickGrabConfig, AnalysisResult, AnalysisReport,
    AnalysisVerdict, ReportFormat, CommandRiskLevel, JavaScriptRedirectChain
)
from redirect_follower import collect_redirects

DEFAULT_CLICKFIX_GIST_ID = "9f563dfb78a06fad5db794f33ba93a3f"
DEFAULT_CLICKFIX_GIST_FILENAME = "clickfix_domains.txt"
CLICKFIX_GIST_CACHE_FILE = pathlib.Path("analysis/clickfix_gist_cache.json")

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("clickgrab")

BASE_DIR = Path(__file__).resolve().parent.parent

def load_environment() -> None:
    """Load environment variables from config/env or env file.
    
    Returns:
        Optional[str]: OTX API key if found, None otherwise
    """
    # Try loading from config/env first
    config_env_path = BASE_DIR / "config" / "env"
    if os.path.exists(config_env_path):
        load_dotenv(config_env_path)
    else:
        # Fall back to env in root directory
        load_dotenv()


def sanitize_url(url: str) -> str:
    """Clean up defanged URLs to make them processable.
    
    Common defanging patterns in threat intelligence:
    - [.] -> .
    - [:]  -> :
    - hxxp -> http
    - hxxps -> https
    - (:) -> :
    - (.) -> .
    
    Args:
        url: The potentially defanged URL
        
    Returns:
        str: Sanitized URL ready for processing
    """
    if not url:
        return url
    
    # Make a copy to work with
    sanitized = url.strip()
    
    # Remove common defanging patterns
    defang_patterns = [
        ('[.]', '.'),      # [.] -> .
        ('[:]', ':'),      # [:] -> :
        ('(.)', '.'),      # (.) -> .
        ('(:)', ':'),      # (:) -> :
        ('[://]', '://'),  # [://] -> ://
        ('hxxp://', 'http://'),   # hxxp:// -> http://
        ('hxxps://', 'https://'), # hxxps:// -> https://
        ('hXXp://', 'http://'),   # hXXp:// -> http://
        ('hXXps://', 'https://'), # hXXps:// -> https://
    ]
    
    for pattern, replacement in defang_patterns:
        sanitized = sanitized.replace(pattern, replacement)
    
    # Log if we made changes
    if sanitized != url:
        logger.info(f"URL defanged: '{url}' -> '{sanitized}'")
    
    return sanitized


def _read_text_capped(response, max_bytes: int = 3_000_000, deadline: Optional[float] = None) -> str:
    """Read a streamed response body with a hard byte cap AND wall-clock deadline.

    requests' read timeout only bounds the gap *between* chunks, so a host that
    trickles one byte just under each read-timeout boundary can keep a single GET
    alive indefinitely. Reading via ``iter_content`` against a ``time.monotonic``
    deadline bounds total transfer time regardless of how the server behaves.
    """
    import time as _t
    chunks = []
    total = 0
    try:
        for chunk in response.iter_content(8192):
            if chunk:
                chunks.append(chunk)
                total += len(chunk)
            if total >= max_bytes:
                break
            if deadline is not None and _t.monotonic() > deadline:
                logger.debug("Body read hit wall-clock deadline; returning partial content.")
                break
    except requests.RequestException as exc:
        logger.debug(f"Stream read interrupted: {exc}")
    raw = b"".join(chunks)
    enc = response.encoding or "utf-8"
    try:
        return raw.decode(enc, errors="ignore")
    except (LookupError, TypeError):
        return raw.decode("utf-8", errors="ignore")


def get_html_content(
    url: str,
    proxies: Dict[str, str] | None,
    max_redirects: int = 2,
    headers: Dict[str, str] | None = None,
) -> Optional[str]:
    """Fetch HTML content from a URL.
    
    Args:
        url: The URL to fetch content from
        max_redirects: Maximum number of redirects to follow
        headers: Optional headers merged over the defaults; overriding User-Agent drops the Sec-CH-UA* hints
        
    Returns:
        str: HTML content if successful, None otherwise
    """
    try:
        # Check if URL is from a CDN known to host malware
        suspicious_cdns = [
            'cdn.jsdelivr.net',
            'code.jquery.com',
            'unpkg.com',  # Another potentially abused CDN
            'stackpath.bootstrapcdn.com'  # Also potentially abused
        ]
        
        parsed_url = urlparse(url)
        if any(cdn in parsed_url.netloc.lower() for cdn in suspicious_cdns):
            logger.warning(f"URL {url} is from a CDN known to host malware. Proceeding with analysis...")

        default_headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Sec-CH-UA': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
            'Sec-CH-UA-Mobile': '?0',
            'Sec-CH-UA-Platform': '"Windows"',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
        }
        if headers and 'User-Agent' in headers:
            # Sec-CH-UA* describe the default Windows Chrome and would contradict another UA
            default_headers = {k: v for k, v in default_headers.items() if not k.lower().startswith('sec-ch-ua')}
        headers = {**default_headers, **(headers or {})}
        
        # Create a session to handle redirects
        import time as _time
        body_deadline = _time.monotonic() + 20  # total wall-clock cap for this fetch
        with requests.Session() as session:
            session.max_redirects = max_redirects
            # (connect, read) timeouts bound each socket op; stream=True + a capped,
            # deadline-bounded body read bounds TOTAL transfer time, so a host that
            # trickles bytes forever can't freeze a scan (slowloris-style C2).
            response = session.get(
                url, headers=headers, timeout=(5, 15),
                allow_redirects=True, verify=False, stream=True, proxies=proxies
            )
            response.raise_for_status()

            # Log redirect chain if any redirects occurred
            if len(response.history) > 0:
                logger.info(f"Redirect chain for {url}:")
                for r in response.history:
                    logger.info(f"  {r.status_code}: {r.url}")
                logger.info(f"  Final URL: {response.url}")

            return _read_text_capped(response, max_bytes=3_000_000, deadline=body_deadline)
    except requests.RequestException as e:
        logger.error(f"Error fetching URL {url}: {e}")
        return None


def download_urlhaus_data(limit: Optional[int] = None, tags: Optional[List[str]] = None) -> List[str]:
    """Download online URLs from URLhaus.
    
    Prefers the authenticated API (if env var URLHAUS_AUTH_KEY or URLHAUS_API_KEY is set),
    and falls back to the public CSV feed otherwise.
    
    Args:
        limit: Maximum number of URLs to return
        tags: List of tags to filter by (e.g. ['FakeCaptcha', 'ClickFix', 'click'])
        
    Returns:
        List[str]: List of URLs matching the criteria
    """
    if tags is None:
        tags = ['FakeCaptcha', 'ClickFix', 'click', 'fakecloudflarecaptcha']

    # 1) Try authenticated API first if a key is present
    import os
    api_key = os.getenv('URLHAUS_AUTH_KEY') or os.getenv('URLHAUS_API_KEY')
    if api_key:
        try:
            # Collect from both recent and tag endpoints
            use_limit = max(1, min(int(limit) if limit else 1000, 1000))
            combined_entries = []

            # 1) Recent
            recent_url = f"https://urlhaus-api.abuse.ch/v1/urls/recent/limit/{use_limit}/"
            logger.info("Fetching URLhaus recent URLs via Auth API…")
            resp = requests.get(recent_url, headers={"Auth-Key": api_key}, timeout=30)
            resp.raise_for_status()
            data_recent = resp.json()
            if data_recent.get('query_status') != 'ok':
                logger.warning(f"URLhaus API recent status: {data_recent.get('query_status')}")
            combined_entries.extend(data_recent.get('urls', []))

            # 2) Tag queries (historical)
            tag_endpoint = "https://urlhaus-api.abuse.ch/v1/tag/"
            seen_urls = set(e.get('url') for e in combined_entries if isinstance(e, dict))
            for tg in set(t.lower() for t in tags):
                try:
                    resp_t = requests.post(
                        tag_endpoint,
                        headers={"Auth-Key": api_key},
                        data={"tag": tg, "limit": str(use_limit)},
                        timeout=30,
                    )
                    resp_t.raise_for_status()
                    data_t = resp_t.json()
                    if data_t.get('query_status') != 'ok':
                        logger.debug(f"Tag query for '{tg}' returned status {data_t.get('query_status')}")
                        continue
                    for e in data_t.get('urls', []):
                        u = e.get('url')
                        if u and u not in seen_urls:
                            combined_entries.append(e)
                            seen_urls.add(u)
                except Exception as te:
                    logger.debug(f"URLhaus tag query error for '{tg}': {te}")

            # Filter combined entries
            urls: List[str] = []
            total_processed = 0
            for entry in combined_entries:
                total_processed += 1
                entry_url = entry.get('url', '')
                entry_tags = entry.get('tags') or []
                entry_tags_lc = [t.lower() for t in entry_tags if isinstance(t, str)]
                url_status = (entry.get('url_status') or '').lower()

                logger.debug(f"\nProcessing API entry #{total_processed}:")
                logger.debug(f"  URL: {entry_url}")
                logger.debug(f"  Tags: {entry_tags_lc}")
                logger.debug(f"  Status: {url_status}")

                # Tag filter
                matching_tags = [tag for tag in tags if tag.lower() in entry_tags_lc]
                if not matching_tags:
                    continue

                # Only consider online pages likely to be HTML landing pages
                if url_status != 'online':
                    continue
                if not (entry_url.endswith('/') or entry_url.endswith('html') or entry_url.endswith('htm')):
                    continue

                urls.append(entry_url)
                if limit and len(urls) >= limit:
                    break

            if urls:
                logger.info(f"Found {len(urls)} matching URLs from {total_processed} API entries (recent+tag)")
                return urls
            else:
                logger.info("No matching URLs found via API (recent+tag), falling back to CSV feed…")
        except Exception as e:
            logger.error(f"URLhaus API error: {e}. Falling back to CSV feed…")

    # 2) Fallback to CSV feed
    csv_url = "https://urlhaus.abuse.ch/downloads/csv_online" 
    try:
        logger.info("Downloading URL data from URLhaus CSV feed…")
        response = requests.get(csv_url, timeout=30)
        response.raise_for_status()
        lines = response.text.split('\n')
        header_idx = next(i for i, line in enumerate(lines) if line.startswith('# id'))
        clean_header = lines[header_idx].replace('# ', '')
        csv_data = [clean_header] + [line for line in lines[header_idx + 1:] if line and not line.startswith('#')]
        reader = csv.DictReader(csv_data)

        urls = []
        total_processed = 0
        for row in reader:
            if not row:
                continue
            total_processed += 1
            entry_url = row['url']
            url_tags = row['tags'].lower()
            threat = row.get('threat', '')

            logger.debug(f"\nProcessing CSV entry #{total_processed}:")
            logger.debug(f"  URL: {entry_url}")
            logger.debug(f"  Tags: {url_tags}")
            logger.debug(f"  Threat: {threat}")

            matching_tags = [tag for tag in tags if tag.lower() in url_tags]
            if matching_tags and (entry_url.endswith('/') or entry_url.endswith('html') or entry_url.endswith('htm')):
                urls.append(entry_url)
                if limit and len(urls) >= limit:
                    break

        logger.info(f"Found {len(urls)} matching URLs from {total_processed} CSV entries")
        return urls
    except Exception as e:
        logger.error(f"Error downloading URLhaus CSV data: {e}")
        return []


def get_latest_gist_url(gist_id: str, filename: str) -> Optional[str]:
    """Fetch the latest raw URL for a specific file in a GitHub Gist using the API.
    
    This ensures we always get the most recent version even if the gist is updated.
    
    Args:
        gist_id: The GitHub Gist ID
        filename: The name of the file within the gist
        
    Returns:
        Optional[str]: The raw URL to the latest version, or None if unavailable
    """
    try:
        api_url = f"https://api.github.com/gists/{gist_id}"
        logger.debug(f"Fetching latest gist metadata from GitHub API: {api_url}")
        
        headers = {
            'Accept': 'application/vnd.github.v3+json',
            'User-Agent': 'ClickGrab-URLAnalyzer'
        }
        
        response = requests.get(api_url, headers=headers, timeout=30)
        response.raise_for_status()
        
        gist_data = response.json()
        
        # Get the file data from the gist
        if 'files' in gist_data and filename in gist_data['files']:
            file_data = gist_data['files'][filename]
            raw_url = file_data.get('raw_url')
            
            if raw_url:
                logger.info(f"Retrieved latest raw URL for {filename}")
                return raw_url
            else:
                logger.warning(f"No raw_url found for {filename} in gist {gist_id}")
                return None
        else:
            logger.warning(f"File {filename} not found in gist {gist_id}")
            return None
            
    except requests.RequestException as exc:
        logger.warning(f"Could not fetch gist metadata from API: {exc}. Falling back to direct URL.")
        # Fall back to the standard raw URL format (always points to latest)
        return f"https://gist.githubusercontent.com/{gist_id}/raw/{filename}"
    except Exception as exc:
        logger.warning(f"Unexpected error fetching gist metadata: {exc}")
        return f"https://gist.githubusercontent.com/{gist_id}/raw/{filename}"


def fetch_clickfix_gist(gist_id: str = DEFAULT_CLICKFIX_GIST_ID, filename: str = DEFAULT_CLICKFIX_GIST_FILENAME) -> Tuple[List[str], Optional[str]]:
    """Fetch domain list from the public ClickFix gist feed.

    The gist is a simple pipe-delimited table with one domain per row. The
    function normalizes the entries, deduplicates them, and returns both the
    list of domains and a hash of the raw content to allow change detection.

    Args:
        gist_id: The GitHub Gist ID
        filename: The filename within the gist

    Returns:
        Tuple[List[str], Optional[str]]: unique domain list and optional SHA256 hash.
    """
    try:
        # Get the latest raw URL from the GitHub API
        url = get_latest_gist_url(gist_id, filename)
        
        if not url:
            logger.error("Could not determine gist URL")
            return [], None
        
        logger.info(f"Fetching ClickFix gist feed from {url}")
        response = requests.get(url, timeout=30)
        response.raise_for_status()

        raw_text = response.text
        content_hash = hashlib.sha256(raw_text.encode("utf-8")).hexdigest()

        domains: List[str] = []
        seen: set[str] = set()

        # Parse line by line - handle both plain text and pipe-delimited formats
        for line in raw_text.splitlines():
            line = line.strip()
            
            # Skip empty lines
            if not line:
                continue
            
            # Skip markdown table headers and separators
            if line.startswith('|') and ('-' in line or 'domain' in line.lower()):
                continue
            
            # Handle pipe-delimited format (| domain |) or plain format
            if line.startswith('|'):
                cleaned = line.strip('|').strip()
            else:
                cleaned = line
            
            if not cleaned:
                continue

            # Domains might include URLs; attempt to normalize
            domain = cleaned
            if domain.startswith("http"):
                try:
                    parsed = urlparse(domain)
                    domain = parsed.netloc or parsed.path
                except Exception:
                    logger.debug(f"Could not parse domain entry '{cleaned}'")
                    continue

            domain = domain.strip()
            if not domain:
                continue

            domain = domain.lower()
            if domain not in seen:
                seen.add(domain)
                domains.append(domain)

        logger.info(f"Fetched {len(domains)} unique domains from ClickFix gist")
        return domains, content_hash

    except requests.RequestException as exc:
        logger.error(f"Error fetching ClickFix gist feed: {exc}")
        return [], None


def load_clickfix_cache(gist_id: str) -> Tuple[Optional[str], List[str]]:
    """Load cached ClickFix gist metadata from disk."""
    if not CLICKFIX_GIST_CACHE_FILE.exists():
        return None, []

    try:
        with CLICKFIX_GIST_CACHE_FILE.open("r", encoding="utf-8") as cache_file:
            data = json.load(cache_file)

        if not isinstance(data, dict):
            return None, []

        cached_gist_id = data.get("gist_id")
        if cached_gist_id and cached_gist_id != gist_id:
            logger.info(f"Cached gist ID {cached_gist_id} differs from requested {gist_id}, ignoring cache")
            return None, []

        cached_hash = data.get("hash")
        cached_domains = data.get("domains") or []

        if not isinstance(cached_domains, list):
            cached_domains = []

        # Deduplicate while preserving order
        deduped_domains = list(dict.fromkeys(str(domain).lower().strip() for domain in cached_domains if domain))
        return cached_hash, deduped_domains

    except Exception as exc:
        logger.warning(f"Could not read ClickFix gist cache: {exc}")
        return None, []


def save_clickfix_cache(hash_value: str, domains: List[str], gist_id: str) -> None:
    """Persist ClickFix gist metadata so future runs can skip unchanged feeds."""
    try:
        CLICKFIX_GIST_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
        payload = {
            "gist_id": gist_id,
            "hash": hash_value,
            "domains": domains,
            "cached_at": datetime.utcnow().isoformat()
        }
        with CLICKFIX_GIST_CACHE_FILE.open("w", encoding="utf-8") as cache_file:
            json.dump(payload, cache_file, indent=2)
    except Exception as exc:
        logger.warning(f"Failed to write ClickFix gist cache: {exc}")


def build_urls_from_domains(domains: List[str]) -> List[str]:
    """Convert bare domains into full URLs suitable for analysis."""
    urls: List[str] = []
    seen: set[str] = set()

    for domain in domains:
        if not domain:
            continue

        sanitized = sanitize_url(domain.strip())
        if not sanitized:
            continue

        if not sanitized.startswith(("http://", "https://")):
            sanitized = f"https://{sanitized}"

        sanitized = sanitized.rstrip("/")

        if sanitized not in seen:
            seen.add(sanitized)
            urls.append(sanitized)

    return urls


def collect_clickfix_gist_urls(config: ClickGrabConfig) -> List[str]:
    """Return newly observed ClickFix gist domains as URLs."""
    # Use custom gist_id if provided, otherwise use default
    gist_id = config.clickfix_gist_id or DEFAULT_CLICKFIX_GIST_ID
    filename = DEFAULT_CLICKFIX_GIST_FILENAME
    
    logger.info(f"Collecting domains from ClickFix gist ID: {gist_id}")
    domains, new_hash = fetch_clickfix_gist(gist_id, filename)

    if not domains:
        logger.warning("No domains retrieved from ClickFix gist feed")
        return []

    cached_hash, cached_domains = load_clickfix_cache(gist_id)

    if new_hash and cached_hash and new_hash == cached_hash:
        logger.info("ClickFix gist has not changed since the last run. Skipping analysis of cached domains.")
        return []

    cached_domain_set = set(cached_domains)
    new_domains = [domain for domain in domains if domain not in cached_domain_set]

    if not new_domains:
        logger.info("No new ClickFix domains detected compared to cached run.")
    else:
        logger.info(f"Identified {len(new_domains)} new ClickFix domains for analysis")

    if new_hash:
        save_clickfix_cache(new_hash, domains, gist_id)

    if not new_domains:
        return []

    return build_urls_from_domains(new_domains)


def download_otx_data(limit: Optional[int] = None, tags: Optional[List[str]] = None, days: int = 30) -> List[str]:
    """Download URLs from AlienVault OTX.
    
    Args:
        limit: Maximum number of URLs to return
        tags: List of tags to filter by (e.g. ['FakeCaptcha', 'ClickFix', 'click'])
        days: Number of days to look back for indicators
        
    Returns:
        List[str]: List of URLs matching the criteria
    """
    try:
        logger.info(f"Downloading URL data from AlienVault OTX (past {days} days)...")
        
        # Get API key from environment
        api_key = os.getenv('OTX_API_KEY')
        if not api_key:
            logger.error("OTX API key not found. Please set OTX_API_KEY in config/env or env file")
            return []

        if tags is None:
            tags = ['FakeCaptcha', 'ClickFix', 'click', 'fakecloudflarecaptcha']
        
        results = []
        
        try:
            # Process each tag
            for tag in tags:
                logger.debug(f"Searching for indicators with tag: {tag}")
                
                # Build initial query URL - similar to PowerShell approach
                query = f"{tag.lower()} modified:<{days}d"
                otx_query = f"https://otx.alienvault.com/otxapi/indicators?include_inactive=0&sort=-modified&page=1&limit=100&q={query}&type=URL"
                
                page_count = 1
                
                # Use pagination like in PowerShell script
                while otx_query:
                    logger.debug(f"Fetching page {page_count} from AlienVault OTX...")
                    
                    # Make request with API key
                    headers = {'X-OTX-API-KEY': api_key}
                    response = requests.get(otx_query, headers=headers)
                    response.raise_for_status()
                    data = response.json()
                    
                    # Process indicators from this page
                    if 'results' in data:
                        for item in data['results']:
                            url = item.get('indicator')
                            if url and (url.endswith('/') or url.endswith('html') or url.endswith('htm')):
                                # Get additional metadata for the URL
                                try:
                                    meta_url = f"https://otx.alienvault.com/api/v1/indicators/url/{url}/url_list"
                                    meta_response = requests.get(meta_url, headers=headers)
                                    meta_response.raise_for_status()
                                    meta_data = meta_response.json()
                                    
                                    # Log metadata if available
                                    if isinstance(meta_data, dict) and meta_data.get('url_list'):
                                        logger.debug(f"Found URL with metadata: {url}")
                                        if isinstance(meta_data['url_list'], list) and meta_data['url_list']:
                                            first_entry = meta_data['url_list'][0]
                                            logger.debug(f"  Added: {first_entry.get('date')}")
                                    else:
                                        logger.debug(f"No metadata available for {url}")
                                except Exception as e:
                                    logger.debug(f"Could not fetch metadata for {url}: {str(e)}")
                                
                                if url not in results:
                                    results.append(url)
                                    logger.debug(f"Added URL: {url}")
                                    
                                    # Check limit
                                    if limit and len(results) >= limit:
                                        logger.debug(f"Reached limit of {limit} URLs from OTX")
                                        return results[:limit]
                    
                    # Get next page URL if available
                    otx_query = data.get('next')
                    page_count += 1
                    
                    logger.debug(f"Downloaded {len(results)} URLs so far...")
                        
        except Exception as e:
            logger.error(f"Error fetching OTX indicators: {e}")
            return results
        
        logger.info(f"Found {len(results)} matching URLs from AlienVault OTX")
        return results
        
    except Exception as e:
        logger.error(f"Error downloading AlienVault OTX data: {e}")
        return []


def fetch_and_analyze_external_js(base_url: str, html_content: str, proxies) -> List[str]:
    """Fetch external JavaScript files and analyze them for obfuscation.
    
    This ensures we don't claim "heavy obfuscation" without actually
    looking at the JS content.
    
    Args:
        base_url: The base URL for resolving relative paths
        html_content: The HTML content containing script tags
        
    Returns:
        List of obfuscation indicators found in external JS files
    """
    from bs4 import BeautifulSoup
    from urllib.parse import urljoin
    
    results = []
    
    try:
        soup = BeautifulSoup(html_content, "html.parser")
        
        # Find external script tags
        external_scripts = []
        for script_tag in soup.find_all("script", src=True):
            raw_src = script_tag.get("src")
            if raw_src:
                script_url = urljoin(base_url, raw_src.strip())
                if script_url not in external_scripts:
                    external_scripts.append(script_url)
        
        if not external_scripts:
            return results

        # Limit to first few external scripts AND a wall-clock budget so this
        # stage can never dominate a scan (some hosts trickle bytes forever).
        import time as _time
        _budget_start = _time.monotonic()
        for script_url in external_scripts[:4]:
            if _time.monotonic() - _budget_start > 12:
                logger.debug("External JS analysis budget (12s) exceeded; stopping early.")
                break
            try:
                logger.debug(f"Fetching external JS for obfuscation analysis: {script_url}")
                response = requests.get(
                    script_url,
                    timeout=(4, 6),
                    verify=False,
                    stream=True,
                    headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"},
                    proxies=proxies,
                )
                response.raise_for_status()

                # Only analyze first 50KB, bounded by the remaining stage budget so a
                # trickling host can't stall mid-download.
                js_content = _read_text_capped(
                    response, max_bytes=50000, deadline=_budget_start + 12
                )
                
                # Run obfuscation detection on the actual JS content
                obfuscation_hits = extractors.extract_heavy_obfuscation(js_content)
                
                for hit in obfuscation_hits:
                    # Prefix with the source URL so we know where it came from
                    results.append(f"[External JS: {script_url}] {hit}")
                
                logger.debug(f"Found {len(obfuscation_hits)} obfuscation patterns in {script_url}")
                
            except requests.RequestException as e:
                logger.debug(f"Failed to fetch external JS {script_url}: {e}")
                continue
            except Exception as e:
                logger.debug(f"Error analyzing external JS {script_url}: {e}")
                continue
                
    except Exception as e:
        logger.debug(f"Error in external JS analysis: {e}")
    
    return results


def analyze_url(
    url: str,
    proxies: Dict[str, str] | None = None,
    headers: Dict[str, str] | None = None,
) -> Optional[AnalysisResult]:
    """Analyze a URL for malicious content and return results as a Pydantic model.
    
    Args:
        url: The URL to analyze
        headers: Optional request headers merged over the defaults (e.g. a custom User-Agent)
        
    Returns:
        Optional[AnalysisResult]: Analysis results if successful, None otherwise
    """
    logger.info(f"Analyzing URL: {url}")
    
    # Sanitize URL to remove common defanging patterns
    url = sanitize_url(url)
    
    # If the URL doesn't start with http:// or https://, assume https://
    if not url.startswith(('http://', 'https://')):
        url = 'https://' + url
    
    # Create base analysis result
    result = AnalysisResult(
        URL=url,
        RawHTML=""
    )

    # Get HTML content
    html_content = get_html_content(url, proxies=proxies, headers=headers)
    if not html_content:
        logger.error(f"Failed to retrieve content from {url}")
        # Still return a result with empty content and failed status
        result.RawHTML = "ERROR: Failed to retrieve content"
        result.SuspiciousKeywords = ["failed_to_retrieve"]
        # Mark as non-suspicious since we couldn't analyze it
        return result
    
    # Update with actual content
    result.RawHTML = html_content

    # Extract various indicators using optimized extractor functions
    result.Base64Strings = extractors.extract_base64_strings(html_content)
    result.URLs = extractors.extract_urls(html_content)
    result.PowerShellCommands = extractors.extract_powershell_commands(html_content)
    result.MshtaCommands = extractors.extract_mshta_commands(html_content)
    result.ConhostCommands = extractors.extract_conhost_commands(html_content)
    result.EncodedPowerShell = extractors.extract_encoded_powershell(html_content)
    result.IPAddresses = extractors.extract_ip_addresses(html_content)
    result.ClipboardCommands = extractors.extract_clipboard_commands(html_content)
    result.SuspiciousKeywords = extractors.extract_suspicious_keywords(html_content)
    result.ClipboardManipulation = extractors.extract_clipboard_manipulation(html_content)
    result.PowerShellDownloads = extractors.extract_powershell_downloads(html_content)
    result.CaptchaElements = extractors.extract_captcha_elements(html_content)
    result.ObfuscatedJavaScript = extractors.extract_obfuscated_javascript(html_content)
    result.EtherhidingJavaScript = extractors.extract_etherhiding_payload(html_content, proxies=proxies)
    result.SuspiciousCommands = extractors.extract_suspicious_commands(html_content)

    # Add new extractions
    result.BotDetection = extractors.extract_bot_detection(html_content)
    result.SessionHijacking = extractors.extract_session_hijacking(html_content)
    result.ProxyEvasion = extractors.extract_proxy_evasion(html_content)
    result.JavaScriptRedirects = extractors.extract_js_redirects(html_content)
    result.Base64XoredJavaScript = extractors.extract_base64_and_xored_js(html_content)
    result.ParkingPageLoaders = extractors.extract_parking_page_loaders(html_content)

    # Add November 2025 threat extractions (fake video conferencing, ClickFix instructions, steganography)
    result.FakeVideoConferencing = extractors.extract_fake_video_conferencing(html_content)
    result.ClickFixInstructions = extractors.extract_clickfix_instructions(html_content)
    result.SteganographyIndicators = extractors.extract_steganography_indicators(html_content)
    result.FakeWindowsUpdate = extractors.extract_fake_windows_update(html_content)
    result.FakeCloudflare = extractors.extract_fake_cloudflare(html_content)
    result.HeavyObfuscation = extractors.extract_heavy_obfuscation(html_content)

    # Add December 2025 threat extractions (macOS attacks, AI chat links, ErrTraffic toolkit)
    result.MacOSTerminalCommands = extractors.extract_macos_terminal_commands(html_content)
    result.SharedAIChatLinks = extractors.extract_shared_ai_chat_links(html_content)
    result.WinHttpVBScript = extractors.extract_winhttp_vbscript(html_content)
    result.FakeGlitchLures = extractors.extract_fake_glitch_lures(html_content)
    result.HexEncodedIPs = extractors.extract_hex_encoded_ips(html_content)

    # Add 2026 threat extractions (DNS ClickFix, Windows Terminal, WebDAV, CrashFix, etc.)
    result.DNSClickFix = extractors.extract_dns_clickfix(html_content)
    result.WindowsTerminalClickFix = extractors.extract_windows_terminal_clickfix(html_content)
    result.WebDAVClickFix = extractors.extract_webdav_clickfix(html_content)
    result.FingerExeAbuse = extractors.extract_finger_exe_abuse(html_content)
    result.ConsentFixIndicators = extractors.extract_consentfix_indicators(html_content)
    result.FakeSoftwareDownloads = extractors.extract_fake_software_downloads(html_content)
    result.LLMArtifactAbuse = extractors.extract_llm_artifact_abuse(html_content)

    # 2026: DriveSurge / zTDS (Silent Push) — TDS loader injection + FakeUpdates lures
    result.TDSInjection = extractors.extract_tds_injection(html_content)
    result.FakeBrowserUpdate = extractors.extract_fake_browser_update(html_content)

    # Also check external JS files for obfuscation
    external_js_obfuscation = fetch_and_analyze_external_js(url, html_content, proxies)
    if external_js_obfuscation:
        result.HeavyObfuscation.extend(external_js_obfuscation)

    # Collect redirect chains (inline + external + meta) and follow them
    try:
        redirect_findings = collect_redirects(result.URL, html_content, proxies)
        if redirect_findings:
            result.RedirectFollows = redirect_findings
            for finding in redirect_findings:
                if finding.Source == "external_js" and finding.ScriptURL:
                    result.JavaScriptRedirectChains.append(
                        JavaScriptRedirectChain(
                            ScriptURL=finding.ScriptURL,
                            RedirectType=finding.Method,
                            DestinationURL=finding.FinalURL or finding.OriginalURL,
                            Evidence=finding.Evidence,
                        )
                    )
    except Exception as redirect_exc:
        logger.debug(f"Failed to collect redirects for {result.URL}: {redirect_exc}")

    logger.debug(f"Analysis complete for {url}. Found {result.TotalIndicators} indicators.")

    if result.TotalIndicators > 0:
        threat_score = result.ThreatScore
        logger.debug(f"Threat score: {threat_score}")
        if threat_score >= 60:
            logger.warning(f"HIGH THREAT DETECTED in {url} - Score: {threat_score}")
        elif threat_score >= 30:
            logger.warning(f"MEDIUM THREAT DETECTED in {url} - Score: {threat_score}")

    return result


def is_suspicious(result: AnalysisResult) -> bool:
    """Determine if an analysis result indicates a suspicious site.
    
    Args:
        result: The analysis result to check
        
    Returns:
        bool: True if the site is suspicious, False otherwise
    """
    return result.Verdict == AnalysisVerdict.SUSPICIOUS.value


def generate_html_report(results: List[AnalysisResult], config: ClickGrabConfig) -> str:
    """Generate an HTML report from analysis results.
    
    Args:
        results: List of analysis results
        config: ClickGrab configuration
        
    Returns:
        str: Path to the generated HTML report
    """
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    suspicious_count = sum(1 for result in results if result.Verdict == AnalysisVerdict.SUSPICIOUS.value)
    
    # Output directory
    output_dir = config.output_dir
    os.makedirs(output_dir, exist_ok=True)
    
    # HTML report path
    report_name = f"clickgrab_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.html"
    report_path = os.path.join(output_dir, report_name)
    
    # Generate HTML content
    html_content = f"""
    <!DOCTYPE html>
    <html>
    <head>
        <title>ClickGrab - URL Analysis Report</title>
        <style>
            body {{ font-family: Arial, sans-serif; margin: 20px; }}
            h1, h2, h3 {{ color: #333; }}
            .site {{ border: 1px solid #ddd; margin: 10px 0; padding: 10px; border-radius: 5px; }}
            .site.suspicious {{ border-color: #ff9999; background-color: #ffeeee; }}
            .site-url {{ font-weight: bold; }}
            .indicator {{ margin: 5px 0; }}
            .indicator-title {{ font-weight: bold; }}
            .summary {{ background-color: #f5f5f5; padding: 10px; border-radius: 5px; margin-bottom: 20px; }}
            pre {{ background-color: #f8f8f8; padding: 10px; border-radius: 3px; overflow-x: auto; }}
            .highlight {{ background-color: yellow; }}
            .risk-high {{ color: #d9534f; font-weight: bold; }}
            .risk-medium {{ color: #f0ad4e; }}
            .risk-low {{ color: #5bc0de; }}
            .total-indicators {{ font-size: 1.2em; margin-top: 10px; }}
            .score-display {{ 
                display: inline-block; 
                padding: 5px 10px; 
                border-radius: 4px; 
                font-weight: bold; 
                margin-left: 10px;
                color: white;
            }}
            .score-high {{ background-color: #d9534f; }}
            .score-medium {{ background-color: #f0ad4e; }}
            .score-low {{ background-color: #5bc0de; }}
            .score-none {{ background-color: #5cb85c; }}
        </style>
    </head>
    <body>
        <h1>ClickGrab URL Analysis Report</h1>
        <div class="summary">
            <p><strong>Generated:</strong> {timestamp}</p>
            <p><strong>Sites Analyzed:</strong> {len(results)}</p>
            <p><strong>Suspicious Sites:</strong> {suspicious_count}</p>
        </div>
        
        <h2>Analysis Results</h2>
    """
    
    # Add each site analysis
    for result in results:
        is_sus = result.Verdict == AnalysisVerdict.SUSPICIOUS.value
        sus_class = "suspicious" if is_sus else ""
        
        # Determine threat score styling
        threat_score = result.ThreatScore
        score_class = "score-none"
        if threat_score >= 60:
            score_class = "score-high"
        elif threat_score >= 30:
            score_class = "score-medium"
        elif threat_score > 0:
            score_class = "score-low"
            
        html_content += f"""
        <div class="site {sus_class}">
            <h3 class="site-url">{result.URL}</h3>
            <p>
                <strong>Verdict:</strong> {'⚠️ SUSPICIOUS' if is_sus else '✅ Likely Safe'}
                <span class="score-display {score_class}">Score: {threat_score}</span>
            </p>
            <p class="total-indicators"><strong>Total Indicators:</strong> {result.TotalIndicators}</p>
        """
        
        # Base64 Strings
        if result.Base64Strings:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Base64 Strings ({len(result.Base64Strings)})</p>
                <ul>
            """
            for b64 in result.Base64Strings:
                html_content += f"<li><strong>Encoded:</strong> {b64.Base64[:50]}...</li>"
                html_content += f"<li><strong>Decoded:</strong> <pre>{b64.Decoded[:200]}...</pre></li>"
                html_content += f"<li><strong>Contains PowerShell:</strong> {'Yes ⚠️' if b64.ContainsPowerShell else 'No'}</li>"
            html_content += "</ul></div>"
        
        # PowerShell Commands
        if result.PowerShellCommands:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">PowerShell Commands ({len(result.PowerShellCommands)})</p>
                <ul>
            """
            for cmd in result.PowerShellCommands:
                html_content += f"<li><pre>{cmd}</pre></li>"
            html_content += "</ul></div>"

        # Mshta Commands
        if result.MshtaCommands:
            html_content += f"""
                <div class="indicator">
                    <p class="indicator-title">Mshta Commands ({len(result.MshtaCommands)})</p>
                    <ul>
                """
            for cmd in result.MshtaCommands:
                html_content += f"<li><pre>{cmd}</pre></li>"
            html_content += "</ul></div>"

        if result.ConhostCommands:
            html_content += f"""
                <div class="indicator">
                    <p class="indicator-title">Conhost Commands ({len(result.ConhostCommands)})</p>
                    <ul>
                """
            for cmd in result.ConhostCommands:
                html_content += f"<li><pre>{cmd}</pre></li>"
            html_content += "</ul></div>"

        # Encoded PowerShell
        if result.EncodedPowerShell:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Encoded PowerShell ({len(result.EncodedPowerShell)})</p>
                <ul>
            """
            for enc in result.EncodedPowerShell:
                html_content += f"<li><strong>Full Match:</strong> {enc.FullMatch[:100]}...</li>"
                html_content += f"<li><strong>Decoded:</strong> <pre>{enc.DecodedCommand[:200]}...</pre></li>"
                html_content += f"<li><strong>Suspicious Content:</strong> {'Yes ⚠️' if enc.HasSuspiciousContent else 'No'}</li>"
                html_content += f"<li><strong>Risk Level:</strong> <span class='{get_risk_level_class(enc.RiskLevel)}'>{enc.RiskLevel}</span></li>"
            html_content += "</ul></div>"
        
        # PowerShell Downloads
        if result.PowerShellDownloads:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">PowerShell Downloads ({len(result.PowerShellDownloads)})</p>
                <ul>
            """
            for download in result.PowerShellDownloads:
                html_content += f"<li><strong>Full Match:</strong> {download.FullMatch[:100]}...</li>"
                if download.URL:
                    html_content += f"<li><strong>URL:</strong> {download.URL}</li>"
                html_content += f"<li><strong>Risk Level:</strong> <span class='{get_risk_level_class(download.RiskLevel)}'>{download.RiskLevel}</span></li>"
            html_content += "</ul></div>"
        
        # Clipboard Manipulation (truncated for HTML display)
        if result.ClipboardManipulation:
            max_items = 5
            max_length = 1000
            displayed_items = result.ClipboardManipulation[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Clipboard Manipulation (showing {len(displayed_items)} of {len(result.ClipboardManipulation)})</p>
                <ul>
            """
            for clip in displayed_items:
                truncated_clip = clip[:max_length] + "... [truncated]" if len(clip) > max_length else clip
                html_content += f"<li><pre>{truncated_clip}</pre></li>"
            if len(result.ClipboardManipulation) > max_items:
                html_content += f"<li><em>... and {len(result.ClipboardManipulation) - max_items} more entries (see JSON for full data)</em></li>"
            html_content += "</ul></div>"
        
        # Clipboard Commands
        if result.ClipboardCommands:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Clipboard Commands ({len(result.ClipboardCommands)})</p>
                <ul>
            """
            for cmd in result.ClipboardCommands:
                html_content += f"<li><pre>{cmd}</pre></li>"
            html_content += "</ul></div>"
        
        # CAPTCHA Elements
        if result.CaptchaElements:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">CAPTCHA Elements ({len(result.CaptchaElements)})</p>
                <ul>
            """
            for elem in result.CaptchaElements:
                html_content += f"<li><pre>{elem}</pre></li>"
            html_content += "</ul></div>"
        
        # Obfuscated JavaScript (truncated for HTML display)
        if result.ObfuscatedJavaScript:
            max_items = 5
            max_length = 1000
            displayed_items = result.ObfuscatedJavaScript[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Obfuscated JavaScript (showing {len(displayed_items)} of {len(result.ObfuscatedJavaScript)})</p>
                <ul>
            """
            for js in displayed_items:
                if isinstance(js, dict) and 'script' in js:
                    script_truncated = js['script'][:max_length] + "... [truncated]" if len(js['script']) > max_length else js['script']
                    html_content += f"<li><pre>{script_truncated}</pre></li>"
                    if 'score' in js:
                        html_content += f"<li><strong>Obfuscation Score:</strong> {js['score']}</li>"
                else:
                    js_truncated = str(js)[:max_length] + "... [truncated]" if len(str(js)) > max_length else js
                    html_content += f"<li><pre>{js_truncated}</pre></li>"
            if len(result.ObfuscatedJavaScript) > max_items:
                html_content += f"<li><em>... and {len(result.ObfuscatedJavaScript) - max_items} more entries (see JSON for full data)</em></li>"
            html_content += "</ul></div>"

            # EtherHiding JavaScript (truncated for HTML display)
            if result.EtherhidingJavaScript:
                max_items = 2
                max_length = 1000
                displayed_items = result.EtherhidingJavaScript[:max_items]

                html_content += f"""
                <div class="indicator">
                    <p class="indicator-title">EtherHiding JavaScript (showing {len(displayed_items)} of {len(result.EtherhidingJavaScript)})</p>
                    <ul>
                """
                for js in displayed_items:
                    if isinstance(js, dict):
                        html_content += "<li><ul class='etherhiding-details'>"

                        if 'stage' in js and js['stage'] is not None:
                            html_content += f"<li><strong>Stage:</strong> {js['stage']}</li>"
                        if 'malicious_url' in js and js['malicious_url']:
                            html_content += f"<li><strong>Malicious URL:</strong> {js['malicious_url']}</li>"
                        if 'smart_contract' in js and js['smart_contract']:
                            html_content += f"<li><strong>Smart Contract:</strong> <code>{js['smart_contract']}</code></li>"
                        if 'rpc' in js and js['rpc']:
                            html_content += f"<li><strong>RPC:</strong> {js['rpc']}</li>"
                        if 'selector' in js and js['selector']:
                            html_content += f"<li><strong>Selector:</strong> <code>{js['selector']}</code></li>"

                        # Code/Payload fields with truncation
                        if 'payload' in js and js['payload']:
                            payload_str = str(js['payload'])
                            payload_truncated = payload_str[:max_length] + "... [truncated]" if len(
                                payload_str) > max_length else payload_str
                            html_content += f"<li><strong>Payload:</strong> <pre>{payload_truncated}</pre></li>"

                        if 'decoded_payload' in js and js['decoded_payload']:
                            decoded_str = str(js['decoded_payload'])
                            decoded_truncated = decoded_str[:max_length] + "... [truncated]" if len(
                                decoded_str) > max_length else decoded_str
                            html_content += f"<li><strong>Decoded Payload:</strong> <pre>{decoded_truncated}</pre></li>"

                        html_content += "</ul></li>"
                    else:
                        js_truncated = str(js)[:max_length] + "... [truncated]" if len(str(js)) > max_length else str(
                            js)
                        html_content += f"<li><pre>{js_truncated}</pre></li>"

                if len(result.EtherhidingJavaScript) > max_items:
                    html_content += f"<li><em>... and {len(result.EtherhidingJavaScript) - max_items} more entries (see JSON for full data)</em></li>"

                html_content += "</ul></div>"
        
        # Suspicious Commands
        if result.SuspiciousCommands:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Suspicious Commands ({len(result.SuspiciousCommands)})</p>
                <ul>
            """
            for cmd in result.SuspiciousCommands:
                risk_class = get_risk_level_class(cmd.RiskLevel)
                
                html_content += f"<li><strong>Type:</strong> {cmd.CommandType}</li>"
                html_content += f"<li><strong>Risk Level:</strong> <span class='{risk_class}'>{cmd.RiskLevel}</span></li>"
                html_content += f"<li><strong>Command:</strong> <pre>{cmd.Command}</pre></li>"
                if cmd.Source:
                    html_content += f"<li><strong>Source:</strong> {cmd.Source}</li>"
            html_content += "</ul></div>"
        
        # High Risk Commands Summary
        if result.HighRiskCommands:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">⚠️ High Risk Commands Summary ({len(result.HighRiskCommands)})</p>
                <ul>
            """
            for cmd in result.HighRiskCommands:
                html_content += f"<li><strong>{cmd.CommandType}:</strong> <pre>{cmd.Command[:100]}{'...' if len(cmd.Command) > 100 else ''}</pre></li>"
            html_content += "</ul></div>"
        
        # Suspicious Keywords (truncated for HTML display)
        if result.SuspiciousKeywords:
            max_items = 30
            displayed_items = result.SuspiciousKeywords[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Suspicious Keywords (showing {len(displayed_items)} of {len(result.SuspiciousKeywords)})</p>
                <ul>
            """
            for kw in displayed_items:
                html_content += f"<li>{kw}</li>"
            if len(result.SuspiciousKeywords) > max_items:
                html_content += f"<li><em>... and {len(result.SuspiciousKeywords) - max_items} more keywords (see JSON for full data)</em></li>"
            html_content += "</ul></div>"
        
        # URLs (truncated for HTML display)
        if result.URLs:
            max_items = 30
            displayed_items = result.URLs[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">URLs (showing {len(displayed_items)} of {len(result.URLs)})</p>
                <ul>
            """
            for url in displayed_items:
                html_content += f"<li>{url}</li>"
            if len(result.URLs) > max_items:
                html_content += f"<li><em>... and {len(result.URLs) - max_items} more URLs (see JSON for full data)</em></li>"
            html_content += "</ul></div>"
        
        # IP Addresses
        if result.IPAddresses:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">IP Addresses ({len(result.IPAddresses)})</p>
                <ul>
            """
            for ip in result.IPAddresses:
                html_content += f"<li>{ip}</li>"
            html_content += "</ul></div>"
        
        # Add the new extraction fields: Bot Detection, Session Hijacking, Proxy Evasion
        # Bot Detection (truncated for HTML display)
        if result.BotDetection:
            max_items = 10
            max_length = 500
            displayed_items = result.BotDetection[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Bot Detection and Sandbox Evasion (showing {len(displayed_items)} of {len(result.BotDetection)})</p>
                <ul>
            """
            for detection in displayed_items:
                truncated_detection = detection[:max_length] + "... [truncated]" if len(detection) > max_length else detection
                html_content += f"<li><pre>{truncated_detection}</pre></li>"
            if len(result.BotDetection) > max_items:
                html_content += f"<li><em>... and {len(result.BotDetection) - max_items} more entries (see JSON for full data)</em></li>"
            html_content += "</ul></div>"
            
        # Session Hijacking
        if result.SessionHijacking:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Session Hijacking Attempts ({len(result.SessionHijacking)})</p>
                <ul>
            """
            for hijack in result.SessionHijacking:
                html_content += f"<li><pre>{hijack}</pre></li>"
            html_content += "</ul></div>"
            
        # Proxy Evasion (truncated for HTML display)
        if result.ProxyEvasion:
            max_items = 10
            max_length = 500
            displayed_items = result.ProxyEvasion[:max_items]
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Proxy/Security Tool Evasion (showing {len(displayed_items)} of {len(result.ProxyEvasion)})</p>
                <ul>
            """
            for evasion in displayed_items:
                truncated_evasion = evasion[:max_length] + "... [truncated]" if len(evasion) > max_length else evasion
                html_content += f"<li><pre>{truncated_evasion}</pre></li>"
            if len(result.ProxyEvasion) > max_items:
                html_content += f"<li><em>... and {len(result.ProxyEvasion) - max_items} more entries (see JSON for full data)</em></li>"
            html_content += "</ul></div>"
        
        # JavaScript Redirects
        if result.JavaScriptRedirects or result.JavaScriptRedirectChains:
            chain_count = len(result.JavaScriptRedirectChains)
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">JavaScript Redirects and Loaders ({len(result.JavaScriptRedirects) + chain_count})</p>
                <ul>
            """
            for redirect in result.JavaScriptRedirects:
                html_content += f"<li><pre>{redirect}</pre></li>"
            if result.JavaScriptRedirectChains:
                html_content += "<li><strong>External Script Redirect Chains:</strong></li>"
                for chain in result.JavaScriptRedirectChains:
                    html_content += "<li>"
                    html_content += f"<div><strong>Script:</strong> {chain.ScriptURL}</div>"
                    html_content += f"<div><strong>Type:</strong> {chain.RedirectType}</div>"
                    html_content += f"<div><strong>Destination:</strong> {chain.DestinationURL}</div>"
                    html_content += f"<div><pre>{chain.Evidence}</pre></div>"
                    html_content += "</li>"
            html_content += "</ul></div>"

        # Base64 + XOR JavaScript
            # Base64 + XOR JavaScript
            if result.Base64XoredJavaScript:
                html_content += f"""
                    <div class="indicator">
                        <p class="indicator-title risk-high">Base64 and XOR JavaScript ({len(result.Base64XoredJavaScript)})</p>
                        <ul>
                    """

                for js in result.Base64XoredJavaScript:
                    # Handle both dicts and objects safely
                    get_val = lambda key: js.get(key) if isinstance(js, dict) else getattr(js, key, None)

                    script_idx = get_val("script_index")
                    xor_key = get_val("xor_key")
                    matched_xor = get_val("matched_xor_op")
                    matched_b64 = get_val("matched_b64") or ""
                    decrypted_text = get_val("decrypted_text")
                    error = get_val("error")

                    b64_trunc = matched_b64[:100] + ("..." if len(matched_b64) > 100 else "")

                    html_content += "<li>"
                    html_content += f"<div><strong>Script Index:</strong> #{script_idx}</div>"

                    if xor_key:
                        html_content += f"<div><strong>XOR Key:</strong> <code>{xor_key}</code></div>"
                    if matched_xor:
                        html_content += f"<div><strong>XOR Op:</strong> <code>{matched_xor}</code></div>"
                    if matched_b64:
                        html_content += f"<div><strong>Base64 Payload:</strong> <code>{b64_trunc}</code></div>"
                    if decrypted_text:
                        html_content += f"<div><strong>Decrypted Output:</strong> <pre>{decrypted_text}</pre></div>"
                    if error:
                        html_content += f"<div><strong>Error:</strong> <span class='risk-high'>{error}</span></div>"

                    html_content += "</li>"

                html_content += "</ul></div>"

            # Redirect Follows
            if result.RedirectFollows:
                html_content += f"""
                    <div class="indicator">
                        <p class="indicator-title">Redirect Follows ({len(result.RedirectFollows)})</p>
                        <ul>
                    """
                for rf in result.RedirectFollows:
                    html_content += f"<li><strong>{rf.Source} ({rf.Method}):</strong> {rf.OriginalURL}"
                    if rf.FinalURL:
                        html_content += f" → {rf.FinalURL}"
                    html_content += f" [{rf.Status}]</li>"
                html_content += "</ul></div>"


        # Redirect Follows
        if result.RedirectFollows:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Redirect Follows ({len(result.RedirectFollows)})</p>
                <ul>
            """
            for rf in result.RedirectFollows:
                html_content += f"<li><strong>{rf.Source} ({rf.Method}):</strong> {rf.OriginalURL}"
                if rf.FinalURL:
                    html_content += f" → {rf.FinalURL}"
                html_content += f" [{rf.Status}]</li>"
            html_content += "</ul></div>"

        # Parking Page Loaders
        if result.ParkingPageLoaders:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Parking Page Loaders ({len(result.ParkingPageLoaders)})</p>
                <ul>
            """
            for loader in result.ParkingPageLoaders:
                html_content += f"<li><pre>{loader[:500]}</pre></li>"
            html_content += "</ul></div>"

        # Fake Video Conferencing
        if result.FakeVideoConferencing:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Fake Video Conferencing ({len(result.FakeVideoConferencing)})</p>
                <ul>
            """
            for fvc in result.FakeVideoConferencing:
                html_content += f"<li>{fvc}</li>"
            html_content += "</ul></div>"

        # ClickFix Instructions
        if result.ClickFixInstructions:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">ClickFix Instructions ({len(result.ClickFixInstructions)})</p>
                <ul>
            """
            for instr in result.ClickFixInstructions:
                html_content += f"<li>{instr}</li>"
            html_content += "</ul></div>"

        # Steganography Indicators
        if result.SteganographyIndicators:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Steganography Indicators ({len(result.SteganographyIndicators)})</p>
                <ul>
            """
            for steg in result.SteganographyIndicators:
                html_content += f"<li>{steg}</li>"
            html_content += "</ul></div>"

        # Fake Windows Update
        if result.FakeWindowsUpdate:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Fake Windows Update ({len(result.FakeWindowsUpdate)})</p>
                <ul>
            """
            for fwu in result.FakeWindowsUpdate:
                html_content += f"<li>{fwu}</li>"
            html_content += "</ul></div>"

        # Fake Cloudflare
        if result.FakeCloudflare:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Fake Cloudflare Verification ({len(result.FakeCloudflare)})</p>
                <ul>
            """
            for fc in result.FakeCloudflare:
                html_content += f"<li>{fc}</li>"
            html_content += "</ul></div>"

        # Heavy Obfuscation
        if result.HeavyObfuscation:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Heavy Obfuscation ({len(result.HeavyObfuscation)})</p>
                <ul>
            """
            for ho in result.HeavyObfuscation[:10]:
                html_content += f"<li><pre>{ho[:500]}</pre></li>"
            if len(result.HeavyObfuscation) > 10:
                html_content += f"<li><em>... and {len(result.HeavyObfuscation) - 10} more</em></li>"
            html_content += "</ul></div>"

        # macOS Terminal Commands
        if result.MacOSTerminalCommands:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">macOS Terminal Commands ({len(result.MacOSTerminalCommands)})</p>
                <ul>
            """
            for mac in result.MacOSTerminalCommands:
                html_content += f"<li>{mac}</li>"
            html_content += "</ul></div>"

        # Shared AI Chat Links
        if result.SharedAIChatLinks:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Shared AI Chat Links ({len(result.SharedAIChatLinks)})</p>
                <ul>
            """
            for ai in result.SharedAIChatLinks:
                html_content += f"<li>{ai}</li>"
            html_content += "</ul></div>"

        # WinHttp VBScript
        if result.WinHttpVBScript:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">WinHttp VBScript Payloads ({len(result.WinHttpVBScript)})</p>
                <ul>
            """
            for wh in result.WinHttpVBScript:
                html_content += f"<li><pre>{wh[:500]}</pre></li>"
            html_content += "</ul></div>"

        # Fake Glitch Lures
        if result.FakeGlitchLures:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Fake Glitch Lures ({len(result.FakeGlitchLures)})</p>
                <ul>
            """
            for gl in result.FakeGlitchLures:
                html_content += f"<li>{gl}</li>"
            html_content += "</ul></div>"

        # Hex-Encoded IPs
        if result.HexEncodedIPs:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Hex-Encoded IPs ({len(result.HexEncodedIPs)})</p>
                <ul>
            """
            for hip in result.HexEncodedIPs:
                html_content += f"<li>{hip}</li>"
            html_content += "</ul></div>"

        # DNS ClickFix
        if result.DNSClickFix:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">DNS-Based ClickFix ({len(result.DNSClickFix)})</p>
                <ul>
            """
            for dns in result.DNSClickFix:
                html_content += f"<li>{dns}</li>"
            html_content += "</ul></div>"

        # Windows Terminal ClickFix
        if result.WindowsTerminalClickFix:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">Windows Terminal ClickFix ({len(result.WindowsTerminalClickFix)})</p>
                <ul>
            """
            for wt in result.WindowsTerminalClickFix:
                html_content += f"<li>{wt}</li>"
            html_content += "</ul></div>"

        # WebDAV ClickFix
        if result.WebDAVClickFix:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">WebDAV ClickFix ({len(result.WebDAVClickFix)})</p>
                <ul>
            """
            for wd in result.WebDAVClickFix:
                html_content += f"<li>{wd}</li>"
            html_content += "</ul></div>"

        # finger.exe Abuse
        if result.FingerExeAbuse:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">finger.exe Abuse / CrashFix ({len(result.FingerExeAbuse)})</p>
                <ul>
            """
            for fe in result.FingerExeAbuse:
                html_content += f"<li>{fe}</li>"
            html_content += "</ul></div>"

        # ConsentFix
        if result.ConsentFixIndicators:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">ConsentFix OAuth Theft ({len(result.ConsentFixIndicators)})</p>
                <ul>
            """
            for cf in result.ConsentFixIndicators:
                html_content += f"<li>{cf}</li>"
            html_content += "</ul></div>"

        # Fake Software Downloads
        if result.FakeSoftwareDownloads:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title">Fake Software Downloads ({len(result.FakeSoftwareDownloads)})</p>
                <ul>
            """
            for fsd in result.FakeSoftwareDownloads:
                html_content += f"<li>{fsd}</li>"
            html_content += "</ul></div>"

        # LLM Artifact Abuse
        if result.LLMArtifactAbuse:
            html_content += f"""
            <div class="indicator">
                <p class="indicator-title risk-high">LLM/AI Artifact Abuse ({len(result.LLMArtifactAbuse)})</p>
                <ul>
            """
            for llm in result.LLMArtifactAbuse:
                html_content += f"<li>{llm}</li>"
            html_content += "</ul></div>"

        html_content += "</div>"

    html_content += """
    </body>
    </html>
    """
    
    # Write HTML to file
    with open(report_path, 'w', encoding='utf-8') as f:
        f.write(html_content)
    
    return report_path


def get_risk_level_class(risk_level: str) -> str:
    """Get CSS class for a risk level.
    
    Args:
        risk_level: The risk level string
        
    Returns:
        str: CSS class for the risk level
    """
    if CommandRiskLevel.HIGH.value in risk_level or CommandRiskLevel.CRITICAL.value in risk_level:
        return "risk-high"
    elif CommandRiskLevel.MEDIUM.value in risk_level:
        return "risk-medium"
    else:
        return "risk-low"


def generate_json_report(results: List[AnalysisResult], config: ClickGrabConfig) -> str:
    """Generate a JSON report from analysis results.
    
    Args:
        results: List of analysis results
        config: ClickGrab configuration
        
    Returns:
        str: Path to the generated JSON report
    """
    output_dir = config.output_dir
    os.makedirs(output_dir, exist_ok=True)
    
    # Create report structure
    report = AnalysisReport(
        timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
        total_sites_analyzed=len(results),
        summary={
            "suspicious_sites": sum(1 for result in results if result.Verdict == AnalysisVerdict.SUSPICIOUS.value),
            "powershell_commands": sum(len(result.PowerShellCommands) for result in results),
            "mshta_commands": sum(len(result.MshtaCommands) for result in results),
            "conhost_commands": sum(len(result.ConhostCommands) for result in results),
            "base64_strings": sum(len(result.Base64Strings) for result in results),
            "clipboard_manipulation": sum(len(result.ClipboardManipulation) for result in results),
            "captcha_elements": sum(len(result.CaptchaElements) for result in results),
            "high_risk_commands": sum(len(result.HighRiskCommands) for result in results),
            "encoded_powershell": sum(len(result.EncodedPowerShell) for result in results),
            "powershell_downloads": sum(len(result.PowerShellDownloads) for result in results),
            "obfuscated_javascript": sum(len(result.ObfuscatedJavaScript) for result in results),
            "etherhiding_javascript": sum(len(result.EtherhidingJavaScript) for result in results),
            "suspicious_commands": sum(len(result.SuspiciousCommands) for result in results),
            "suspicious_keywords": sum(len(result.SuspiciousKeywords) for result in results),
            "ip_addresses": sum(len(result.IPAddresses) for result in results),
            "clipboard_commands": sum(len(result.ClipboardCommands) for result in results),
            "javascript_redirects": sum(len(result.JavaScriptRedirects) for result in results),
            "base64_xored_javascript": sum(len(result.Base64XoredJavaScript) for result in results),
            "javascript_redirect_chains": sum(len(result.JavaScriptRedirectChains) for result in results),
            "redirect_follows": sum(len(result.RedirectFollows) for result in results),
            # November 2025 threat indicators
            "fake_video_conferencing": sum(len(result.FakeVideoConferencing) for result in results),
            "clickfix_instructions": sum(len(result.ClickFixInstructions) for result in results),
            "steganography_indicators": sum(len(result.SteganographyIndicators) for result in results),
            "fake_windows_update": sum(len(result.FakeWindowsUpdate) for result in results),
            "fake_cloudflare": sum(len(result.FakeCloudflare) for result in results),
            "heavy_obfuscation": sum(len(result.HeavyObfuscation) for result in results),
            # December 2025 threat indicators
            "macos_terminal_commands": sum(len(result.MacOSTerminalCommands) for result in results),
            "shared_ai_chat_links": sum(len(result.SharedAIChatLinks) for result in results),
            "winhttp_vbscript": sum(len(result.WinHttpVBScript) for result in results),
            "fake_glitch_lures": sum(len(result.FakeGlitchLures) for result in results),
            "hex_encoded_ips": sum(len(result.HexEncodedIPs) for result in results),
            # 2026 threat indicators
            "dns_clickfix": sum(len(result.DNSClickFix) for result in results),
            "windows_terminal_clickfix": sum(len(result.WindowsTerminalClickFix) for result in results),
            "webdav_clickfix": sum(len(result.WebDAVClickFix) for result in results),
            "finger_exe_abuse": sum(len(result.FingerExeAbuse) for result in results),
            "consentfix_indicators": sum(len(result.ConsentFixIndicators) for result in results),
            "fake_software_downloads": sum(len(result.FakeSoftwareDownloads) for result in results),
            "llm_artifact_abuse": sum(len(result.LLMArtifactAbuse) for result in results),
            "average_threat_score": round(sum(result.ThreatScore for result in results) / len(results)) if results else 0
        },
        sites=results
    )
    
    # JSON report path
    report_name = f"clickgrab_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
    report_path = os.path.join(output_dir, report_name)
    
    # RawHTML is excluded by default — it's ~1-2 MB per site, the static site
    # never reads it, and Streamlit reads it from in-memory results, not disk.
    # Pass --include-raw-html to keep it (e.g. for offline forensic archives).
    dump_kwargs = {"exclude_none": True, "indent": 2}
    if not config.include_raw_html:
        dump_kwargs["exclude"] = {"sites": {"__all__": {"RawHTML"}}}

    with open(report_path, 'w', encoding='utf-8') as f:
        json_data = report.model_dump_json(**dump_kwargs)
        f.write(json_data)

    latest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "latest_consolidated_report.json")
    with open(latest_path, 'w', encoding='utf-8') as f:
        f.write(json_data)

    return report_path


def generate_csv_report(results: List[AnalysisResult], config: ClickGrabConfig) -> str:
    """Generate a CSV report from analysis results.
    
    Args:
        results: List of analysis results
        config: ClickGrab configuration
        
    Returns:
        str: Path to the generated CSV report
    """
    output_dir = config.output_dir
    os.makedirs(output_dir, exist_ok=True)
    
    # CSV report path
    report_name = f"clickgrab_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
    report_path = os.path.join(output_dir, report_name)
    
    # Define CSV headers
    headers = [
        "URL",
        "Suspicious",
        "Threat Score",
        "Total Indicators",
        "Base64Strings",
        "PowerShellCommands",
        "MshtaCommands",
        "ConhostCommands",
        "EncodedPowerShell",
        "PowerShellDownloads",
        "ClipboardManipulation",
        "ClipboardCommands",
        "CaptchaElements",
        "ObfuscatedJavaScript",
        "EtherhidingJavaScript",
        "SuspiciousCommands",
        "SuspiciousKeywords",
        "IP Addresses",
        "High Risk Commands",
        "JavaScript Redirects",
        "Redirect Follows",
        "FakeVideoConferencing",
        "ClickFixInstructions",
        "Steganography",
        "FakeCloudflare",
        "MacOSTerminal",
        "HexEncodedIPs",
        "DNSClickFix",
        "WindowsTerminal",
        "WebDAVClickFix",
        "FingerExeAbuse",
        "ConsentFix",
        "LLMArtifactAbuse",
    ]

    # Write CSV file
    with open(report_path, 'w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(headers)

        for result in results:
            suspicious = "Yes" if result.Verdict == AnalysisVerdict.SUSPICIOUS.value else "No"
            writer.writerow([
                result.URL,
                suspicious,
                result.ThreatScore,
                result.TotalIndicators,
                len(result.Base64Strings),
                len(result.PowerShellCommands),
                len(result.MshtaCommands),
                len(result.ConhostCommands),
                len(result.EncodedPowerShell),
                len(result.PowerShellDownloads),
                len(result.ClipboardManipulation),
                len(result.ClipboardCommands),
                len(result.CaptchaElements),
                len(result.ObfuscatedJavaScript),
                len(result.EtherhidingJavaScript),
                len(result.SuspiciousCommands),
                len(result.SuspiciousKeywords),
                len(result.IPAddresses),
                len(result.HighRiskCommands),
                len(result.JavaScriptRedirects) + len(result.JavaScriptRedirectChains),
                len(result.RedirectFollows),
                len(result.FakeVideoConferencing),
                len(result.ClickFixInstructions),
                len(result.SteganographyIndicators),
                len(result.FakeCloudflare),
                len(result.MacOSTerminalCommands),
                len(result.HexEncodedIPs),
                len(result.DNSClickFix),
                len(result.WindowsTerminalClickFix),
                len(result.WebDAVClickFix),
                len(result.FingerExeAbuse),
                len(result.ConsentFixIndicators),
                len(result.LLMArtifactAbuse),
            ])
    
    return report_path


def generate_threat_intel_exports(results: List[AnalysisResult], config: ClickGrabConfig) -> List[str]:
    """Generate focused threat intel exports: clipboard commands, download cradles, lure variants.

    Writes dated files into a ``threat_intel/`` subdirectory under the configured
    output dir, plus a ``latest_threat_intel.json`` combined file at the project
    root for easy consumption by the Streamlit UI.

    Args:
        results: List of analysis results
        config: ClickGrab configuration

    Returns:
        List[str]: Paths to generated export files
    """
    intel_dir = os.path.join(config.output_dir, "threat_intel")
    os.makedirs(intel_dir, exist_ok=True)
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    date_str = datetime.now().strftime("%Y-%m-%d")
    paths = []

    # Cap individual snippet length so a single obfuscated JS blob can't
    # singlehandedly push these exports past GitHub's 50 MB warning line.
    MAX_VALUE_LEN = 8192

    def _trim(value):
        if isinstance(value, str) and len(value) > MAX_VALUE_LEN:
            return value[:MAX_VALUE_LEN] + f"\n…[truncated, {len(value) - MAX_VALUE_LEN} more chars]"
        return value

    # --- Clipboard Commands ---
    clipboard_rows = []
    for r in results:
        url = r.URL
        for cmd in r.ClipboardCommands:
            clipboard_rows.append({
                "source_url": url,
                "threat_score": r.ThreatScore,
                "category": "Clipboard Command",
                "value": _trim(cmd),
            })
        for snip in r.ClipboardManipulation:
            clipboard_rows.append({
                "source_url": url,
                "threat_score": r.ThreatScore,
                "category": "Clipboard Manipulation JS",
                "value": _trim(snip),
            })

    if clipboard_rows:
        path = os.path.join(intel_dir, f"clipboard_commands_{ts}.json")
        with open(path, "w", encoding="utf-8") as f:
            json.dump(clipboard_rows, f, indent=2, default=str)
        paths.append(path)

    # --- Download Cradles ---
    cradle_fields = [
        ("PowerShellDownloads", "PowerShell Download"),
        ("PowerShellCommands", "PowerShell Command"),
        ("MshtaCommands", "Mshta Command"),
        ("ConhostCommands", "Conhost Command"),
        ("EncodedPowerShell", "Encoded PowerShell"),
        ("MacOSTerminalCommands", "macOS Terminal Command"),
        ("DNSClickFix", "DNS ClickFix"),
        ("WindowsTerminalClickFix", "Windows Terminal"),
        ("WebDAVClickFix", "WebDAV net use"),
        ("FingerExeAbuse", "finger.exe Abuse"),
        ("WinHttpVBScript", "WinHttp VBScript"),
    ]

    cradle_rows = []
    for r in results:
        url = r.URL
        for field_key, label in cradle_fields:
            items = getattr(r, field_key, [])
            for item in items:
                if hasattr(item, "model_dump"):
                    value = item.model_dump()
                elif isinstance(item, dict):
                    value = item
                else:
                    value = item
                cradle_rows.append({
                    "source_url": url,
                    "threat_score": r.ThreatScore,
                    "category": label,
                    "value": _trim(value),
                })

    if cradle_rows:
        path = os.path.join(intel_dir, f"download_cradles_{ts}.json")
        with open(path, "w", encoding="utf-8") as f:
            json.dump(cradle_rows, f, indent=2, default=str)
        paths.append(path)

    # --- Lure Variants ---
    lure_fields = [
        ("ClickFixInstructions", "ClickFix Instructions"),
        ("FakeCloudflare", "Fake Cloudflare"),
        ("FakeVideoConferencing", "Fake Video Conferencing"),
        ("FakeWindowsUpdate", "Fake Windows Update"),
        ("FakeGlitchLures", "Fake Glitch / Broken Page"),
        ("FakeSoftwareDownloads", "Fake Software Download"),
        ("ConsentFixIndicators", "ConsentFix OAuth Theft"),
        ("LLMArtifactAbuse", "LLM / AI Artifact Abuse"),
        ("SharedAIChatLinks", "Shared AI Chat Links"),
        ("FakeBrowserUpdate", "Fake Browser Update"),
        ("CaptchaElements", "CAPTCHA Elements"),
    ]

    lure_rows = []
    for r in results:
        url = r.URL
        for field_key, label in lure_fields:
            items = getattr(r, field_key, [])
            for item in items:
                lure_rows.append({
                    "source_url": url,
                    "threat_score": r.ThreatScore,
                    "category": label,
                    "value": _trim(item),
                })

    if lure_rows:
        path = os.path.join(intel_dir, f"lure_variants_{ts}.json")
        with open(path, "w", encoding="utf-8") as f:
            json.dump(lure_rows, f, indent=2, default=str)
        paths.append(path)

    # --- Combined summary CSV ---
    summary_rows = []
    for r in results:
        summary_rows.append({
            "URL": r.URL,
            "ThreatScore": r.ThreatScore,
            "Verdict": r.Verdict,
            "ClipboardCommands": len(r.ClipboardCommands),
            "ClipboardManipulation": len(r.ClipboardManipulation),
            "PowerShellDownloads": len(r.PowerShellDownloads),
            "PowerShellCommands": len(r.PowerShellCommands),
            "MshtaCommands": len(r.MshtaCommands),
            "ConhostCommands": len(r.ConhostCommands),
            "EncodedPowerShell": len(r.EncodedPowerShell),
            "MacOSTerminalCommands": len(r.MacOSTerminalCommands),
            "DNSClickFix": len(r.DNSClickFix),
            "WindowsTerminalClickFix": len(r.WindowsTerminalClickFix),
            "WebDAVClickFix": len(r.WebDAVClickFix),
            "FingerExeAbuse": len(r.FingerExeAbuse),
            "WinHttpVBScript": len(r.WinHttpVBScript),
            "ClickFixInstructions": len(r.ClickFixInstructions),
            "FakeCloudflare": len(r.FakeCloudflare),
            "FakeVideoConferencing": len(r.FakeVideoConferencing),
            "FakeWindowsUpdate": len(r.FakeWindowsUpdate),
            "FakeGlitchLures": len(r.FakeGlitchLures),
            "FakeSoftwareDownloads": len(r.FakeSoftwareDownloads),
            "ConsentFixIndicators": len(r.ConsentFixIndicators),
            "LLMArtifactAbuse": len(r.LLMArtifactAbuse),
            "TDSInjection": len(r.TDSInjection),
            "FakeBrowserUpdate": len(r.FakeBrowserUpdate),
            "CaptchaElements": len(r.CaptchaElements),
        })

    if summary_rows:
        path = os.path.join(intel_dir, f"threat_intel_summary_{ts}.csv")
        with open(path, "w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=summary_rows[0].keys())
            writer.writeheader()
            writer.writerows(summary_rows)
        paths.append(path)

    # --- Write combined latest file for Streamlit quick-load ---
    combined = {
        "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
        "date": date_str,
        "total_urls_analyzed": len(results),
        "clipboard_commands": clipboard_rows,
        "download_cradles": cradle_rows,
        "lure_variants": lure_rows,
    }
    latest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "latest_threat_intel.json")
    with open(latest_path, "w", encoding="utf-8") as f:
        json.dump(combined, f, indent=2, default=str)
    paths.append(latest_path)

    return paths


def parse_arguments() -> ClickGrabConfig:
    """Parse command line arguments and return as a Pydantic model.
    
    Returns:
        ClickGrabConfig: Configuration based on command line arguments
    """
    parser = argparse.ArgumentParser(description="ClickGrab - URL Analyzer for detecting fake CAPTCHA sites")
    
    parser.add_argument("analyze", nargs="?", help="URL to analyze or path to a file containing URLs (one per line)")
    parser.add_argument("--limit", type=int, help="Limit the number of URLs to process")
    parser.add_argument("--debug", action="store_true", help="Enable debug output")
    parser.add_argument("--output-dir", default="reports", help="Directory for report output")
    parser.add_argument("--format", choices=["html", "json", "csv", "all"], default="all", help="Report format")
    parser.add_argument("--tags", help="Comma-separated list of tags to look for")
    parser.add_argument("--download", action="store_true", help="Download and analyze URLs from URLhaus")
    parser.add_argument("--otx", action="store_true", help="Download and analyze URLs from AlienVault OTX")
    parser.add_argument("--days", type=int, default=30, help="Number of days to look back in AlienVault OTX (default: 30)")
    parser.add_argument("--clickfix-gist", action="store_true", help="Pull domains from the public ClickFix gist feed")
    parser.add_argument("--clickfix-gist-id", default=None, help=f"Override GitHub Gist ID for the ClickFix feed (default: {DEFAULT_CLICKFIX_GIST_ID})")
    parser.add_argument("--export-intel", action="store_true", help="Generate focused threat intel exports (clipboard commands, download cradles, lure variants)")
    parser.add_argument("--include-raw-html", action="store_true", help="Include full RawHTML in JSON report (off by default; reports run ~150x larger with it on)")

    args = parser.parse_args()
    
    # Convert args to dict and create Pydantic model
    return ClickGrabConfig(**vars(args))


def read_urls_from_file(file_path: str) -> List[str]:
    """Read URLs from a file, one per line.
    
    Args:
        file_path: Path to the file containing URLs
        
    Returns:
        List[str]: List of URLs read from the file
    """
    try:
        with open(file_path, 'r') as f:
            return [line.strip() for line in f if line.strip()]
    except Exception as e:
        logger.error(f"Failed to read URLs from file {file_path}: {e}")
        return []


def main():
    """Main entry point for ClickGrab."""
    # Parse arguments
    config = parse_arguments()
    load_environment()
    proxies = {
        "http": os.getenv("HTTP_PROXY"),
        "https": os.getenv("HTTPS_PROXY")
    }
    logger.info(f"Proxies: {proxies}")
    
    # Configure logging level
    if config.debug:
        logger.setLevel(logging.DEBUG)
        # Also set urllib3 warnings to be displayed in debug mode
        logging.getLogger("urllib3").setLevel(logging.WARNING)
    else:
        # Disable request warnings in normal mode
        import urllib3
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        import warnings
        warnings.filterwarnings("ignore")
    
    # Initialize results list
    results = []
    
    # Determine mode of operation
    if config.download or config.otx or config.clickfix_gist:
        urls = []
        
        # Download from URLhaus if requested
        if config.download:
            logger.info("Running in URLhaus download mode")
            tags = None
            if config.tags:
                # config.tags is already a list due to the validator in the Pydantic model
                tags = config.tags
            
            urlhaus_urls = download_urlhaus_data(config.limit, tags)
            if urlhaus_urls:
                logger.info(f"Downloaded {len(urlhaus_urls)} URLs from URLhaus")
                urls.extend(urlhaus_urls)
        
        # Download from AlienVault OTX if requested
        if config.otx:
            logger.info("Running in AlienVault OTX download mode")
            tags = None
            if config.tags:
                tags = config.tags
            
            otx_urls = download_otx_data(config.limit, tags, config.days)
            if otx_urls:
                logger.info(f"Downloaded {len(otx_urls)} URLs from AlienVault OTX")
                urls.extend(otx_urls)

        # Pull domains from ClickFix gist if requested
        if config.clickfix_gist:
            logger.info("Fetching domains from ClickFix gist feed")
            gist_urls = collect_clickfix_gist_urls(config)
            if gist_urls:
                logger.info(f"Queued {len(gist_urls)} URLs from ClickFix gist for analysis")
                urls.extend(gist_urls)
        
        # Deduplicate URLs
        unique_urls = list(dict.fromkeys(urls))
        if len(unique_urls) < len(urls):
            logger.info(f"Removed {len(urls) - len(unique_urls)} duplicate URLs")
        
        # Apply limit after combining sources if needed
        if config.limit and len(unique_urls) > config.limit:
            unique_urls = unique_urls[:config.limit]
            logger.info(f"Limited to {config.limit} URLs total")
        
        if not unique_urls:
            logger.error("No URLs found from the specified sources matching the criteria")
            sys.exit(1)
        
        # Process each URL
        for url in unique_urls:
            result = analyze_url(url)
            results.append(result)
                
    elif config.analyze:
        # Standard mode - analyze specified URL or file
        if os.path.isfile(config.analyze):
            # Read URLs from file
            urls = read_urls_from_file(config.analyze)
            logger.info(f"Loaded {len(urls)} URLs from file {config.analyze}")
            
            # Apply limit if specified
            if config.limit and config.limit > 0:
                urls = urls[:config.limit]
                logger.info(f"Limited to first {config.limit} URLs")
            
            # Process each URL
            for url in urls:
                result = analyze_url(url)
                results.append(result)
        else:
            # Single URL analysis
            result = analyze_url(config.analyze, proxies=proxies)
            results.append(result)
    else:
        # No URL or file specified, and not in download mode
        print("Error: No URL or file specified.")
        print("Usage: python clickgrab.py [URL or file] [options]")
        print("       python clickgrab.py --download [options] to download from URLhaus")
        print("       python clickgrab.py --otx [options] to download from AlienVault OTX")
        print("For more information, use --help")
        sys.exit(1)
    
    # Generate reports
    if results:
        logger.info(f"Analysis complete. Processing {len(results)} results.")
        
        reports = []
        
        if config.format == ReportFormat.HTML.value or config.format == ReportFormat.ALL.value:
            html_report = generate_html_report(results, config)
            reports.append(("HTML", html_report))
        
        if config.format == ReportFormat.JSON.value or config.format == ReportFormat.ALL.value:
            json_report = generate_json_report(results, config)
            reports.append(("JSON", json_report))
        
        if config.format == ReportFormat.CSV.value or config.format == ReportFormat.ALL.value:
            csv_report = generate_csv_report(results, config)
            reports.append(("CSV", csv_report))
        
        # Print summary
        print("\nAnalysis Summary:")
        print(f"URLs analyzed: {len(results)}")
        suspicious_count = sum(1 for r in results if r.Verdict == AnalysisVerdict.SUSPICIOUS.value)
        print(f"Suspicious sites: {suspicious_count} ({round((suspicious_count / len(results)) * 100, 1)}%)")
        
        high_risk_count = sum(len(r.HighRiskCommands) for r in results)
        if high_risk_count > 0:
            print(f"High risk commands detected: {high_risk_count}")
        
        # Print threat scores
        if len(results) > 0:
            scores = [r.ThreatScore for r in results]
            avg_score = sum(scores) / len(scores)
            max_score = max(scores)
            print(f"Average threat score: {avg_score:.1f}")
            print(f"Maximum threat score: {max_score}")
        
        # Generate threat intel exports if requested
        if config.export_intel:
            intel_paths = generate_threat_intel_exports(results, config)
            if intel_paths:
                reports.extend([("Threat Intel", p) for p in intel_paths])
                clipboard_count = sum(len(r.ClipboardCommands) + len(r.ClipboardManipulation) for r in results)
                cradle_count = sum(
                    len(r.PowerShellDownloads) + len(r.PowerShellCommands) +
                    len(r.MshtaCommands) + len(r.ConhostCommands) +
                    len(r.EncodedPowerShell) + len(r.MacOSTerminalCommands) +
                    len(r.DNSClickFix) + len(r.WindowsTerminalClickFix) +
                    len(r.WebDAVClickFix) + len(r.FingerExeAbuse) +
                    len(r.WinHttpVBScript)
                    for r in results
                )
                lure_count = sum(
                    len(r.ClickFixInstructions) + len(r.FakeCloudflare) +
                    len(r.FakeVideoConferencing) + len(r.FakeWindowsUpdate) +
                    len(r.FakeGlitchLures) + len(r.FakeSoftwareDownloads) +
                    len(r.ConsentFixIndicators) + len(r.LLMArtifactAbuse) +
                    len(r.SharedAIChatLinks) + len(r.CaptchaElements)
                    for r in results
                )
                print(f"\nThreat Intel Export Summary:")
                print(f"  Clipboard commands/scripts: {clipboard_count}")
                print(f"  Download cradles: {cradle_count}")
                print(f"  Lure variant indicators: {lure_count}")

        print("\nReports generated:")
        for report_type, report_path in reports:
            print(f"- {report_type}: {report_path}")
    else:
        logger.warning("No results to generate reports from.")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        logger.info("Operation cancelled by user.")
        sys.exit(0)
    except Exception as e:
        logger.error(f"An unexpected error occurred: {e}", exc_info=True)
        sys.exit(1)