Filter sourced gigs for co-op roles (#436)

* Filter sourced gigs for co-op roles

* Preserve relevant HN leads during cleanup

* Protect relevant and staged leads during filtering

* Reconcile current source roles before cleanup

Author: michaelmwu

apps/admin_dashboard/src/main.tsx

@@ -350,6 +350,7 @@ type JobLeadScrapeThread = {
     empty?: number
     seeking_work?: number
     not_contractor_friendly?: number
+    irrelevant_role?: number
     expired?: number
     invalid_or_duplicate?: number
   }
@@ -366,6 +367,7 @@ type JobLeadScrapeResult = {
   created?: number
   updated?: number
   persisted?: number
+  rejected_existing?: number
   total?: number
   incomplete?: boolean
   incomplete_reason?: string | null
@@ -5773,6 +5775,7 @@ function JobLeadScrapeStatusPanel({
     ["empty", filterReasons?.empty],
     ["seeking work", filterReasons?.seeking_work],
     ["not contractor-friendly", filterReasons?.not_contractor_friendly],
+    ["outside co-op roles", filterReasons?.irrelevant_role],
     ["expired", filterReasons?.expired],
     ["invalid or duplicate", filterReasons?.invalid_or_duplicate],
   ].filter(([, count]) => typeof count === "number" && count > 0) as Array<[string, number]>
@@ -5919,6 +5922,12 @@ function JobLeadScrapeStatusPanel({
                 {result.created || 0} created · {result.updated || 0} updated
               </p>
             ) : null}
+            {result?.rejected_existing ? (
+              <p className="text-muted-foreground">
+                {result.rejected_existing} previously unreviewed, unstaged off-topic leads removed
+                from review.
+              </p>
+            ) : null}
           </>
         ) : null}
         {normalizedStatus === "succeeded" && !hasDiscoveryCounts ? (

apps/api/src/five08/backend/static/dashboard/.vite/manifest.json

@@ -1,6 +1,6 @@
 {
   "index.html": {
-    "file": "assets/index-B-Lkpe-o.js",
+    "file": "assets/index-BHZP3Pxu.js",
     "name": "index",
     "src": "index.html",
     "isEntry": true,

…tatic/dashboard/assets/index-B-Lkpe-o.js …tatic/dashboard/assets/index-BHZP3Pxu.jsapps/api/src/five08/backend/static/dashboard/assets/index-B-Lkpe-o.js renamed to apps/api/src/five08/backend/static/dashboard/assets/index-BHZP3Pxu.js apps/api/src/five08/backend/static/dashboard/assets/index-B-Lkpe-o.js renamed to apps/api/src/five08/backend/static/dashboard/assets/index-BHZP3Pxu.js

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>508 Operations Dashboard</title>
-    <script type="module" crossorigin src="/dashboard/assets/index-B-Lkpe-o.js"></script>
+    <script type="module" crossorigin src="/dashboard/assets/index-BHZP3Pxu.js"></script>
     <link rel="stylesheet" crossorigin href="/dashboard/assets/index-B5zOzk_X.css">
   </head>
   <body>

apps/api/src/five08/backend/static/dashboard/index.html

@@ -119,6 +119,16 @@ the board's public first page. Titles that describe a contract domain (such as
 smart contracts or contract administration) or explicitly say full-time are
 excluded from this title-only pilot.
 
+All four sources require a clear engineering, product, data, design, marketing,
+or ecommerce role in the job title or HN header before admitting a lead. The
+filter excludes recruiting, sales, clinical, language, and data annotation
+roles even when their descriptions mention technical skills. A sync also marks
+previously unreviewed, unstaged off-topic leads from that source as rejected so
+they leave the review queue. Reviewed, posted, and staged leads remain untouched.
+The source status reports role mismatches and this cleanup separately. Operators
+can restore a rejected lead if the title filter missed a suitable role; later
+syncs preserve that manual override.
+
 Operators can run all four sources together or trigger any source alone. Each
 source has its own latest-run status and counts, and leads show a source label.
 The source filter is applied before the 50-lead page limit.

docs/discord-gig-dashboard.md

@@ -21,6 +21,7 @@
 from five08.job_leads import (
     JobLeadInput,
     existing_job_lead_external_ids,
+    reject_unreviewed_irrelevant_job_leads,
     update_existing_job_lead,
     upsert_job_lead,
 )
@@ -59,6 +60,52 @@
     r"operations|solutions|compliance|officer)\b",
     re.I,
 )
+_COOP_ROLE_RE = re.compile(
+    r"\b(?:"
+    r"(?:software|full[ -]?stack|front[ -]?end|back[ -]?end|web|mobile|"
+    r"platform|infrastructure|cloud|devops|site reliability|security|"
+    r"machine learning|ml|ai|data|analytics|mechanical|electrical|"
+    r"systems?)\s+(?:engineer(?:ing)?|developer|architect)|"
+    r"engineer(?:ing)?|developer|programmer|sre|devops|"
+    r"full[ -]?stack|front[ -]?end|back[ -]?end|data science|"
+    r"data\s+(?:scientist|analyst|architect|specialist)|"
+    r"(?:product|technical product)\s+(?:manager|designer|lead|owner|analyst)|"
+    r"(?:ux|ui|ux/ui|graphic|graphics|visual|brand|web|creative)\s+designer|"
+    r"(?:ux|ui|ux/ui)\s+researcher|"
+    r"(?:digital|performance|growth|product|content|email|social media)?\s*"
+    r"marketing\s+(?:expert|specialist|manager|strategist|consultant|lead)|"
+    r"(?:seo|sem|paid media|media buying)\s+"
+    r"(?:expert|specialist|manager|strategist|consultant|lead)|"
+    r"media\s+(?:buyer|planner)|"
+    r"(?:shopify|e[ -]?commerce|amazon marketplace)\s+"
+    r"(?:expert|specialist|developer|designer|consultant|manager|lead)|"
+    r"(?:tech|technical|engineering)\s+(?:lead|architect)|"
+    r"(?:database|db|business intelligence|bi)\s+(?:administrator|analyst|engineer)"
+    r")\b",
+    re.I,
+)
+_NON_COOP_ROLE_RE = re.compile(
+    r"\b(?:recruit(?:er|ing|ment)|talent acquisition|sourc(?:er|ing)|"
+    r"sales|account manager|copywriter|transcri(?:ber|ption)|"
+    r"data (?:entry|rater|annotator)|ai trainer|"
+    r"nurs(?:e|ing)|physician|therapist|psychologist|counselor)\b",
+    re.I,
+)
+
+
+def _title_is_coop_role(title: str) -> bool:
+    """Keep explicit technical roles; source descriptions often mention unrelated skills."""
+    return any(
+        _COOP_ROLE_RE.search(part) and not _NON_COOP_ROLE_RE.search(part)
+        for part in re.split(r"[|,;]", title)
+    )
+
+
+def _lead_is_coop_role(source_key: str, title: str, body: str) -> bool:
+    """HN's full first line can carry roles beyond its abbreviated display title."""
+    if source_key == HN_WHO_IS_HIRING_SOURCE_KEY and body:
+        return _title_is_coop_role(body.splitlines()[0])
+    return _title_is_coop_role(title)
 
 
 def _title_is_gig(title: str) -> bool:
@@ -338,13 +385,15 @@ class HackerNewsThreadScrapeReport:
     filtered_empty: int = 0
     filtered_seeking_work: int = 0
     filtered_not_contractor_friendly: int = 0
+    filtered_irrelevant_role: int = 0
 
     @property
     def filtered_out(self) -> int:
         return (
             self.filtered_empty
             + self.filtered_seeking_work
             + self.filtered_not_contractor_friendly
+            + self.filtered_irrelevant_role
         )
 
     def payload(self) -> dict[str, Any]:
@@ -362,6 +411,7 @@ def payload(self) -> dict[str, Any]:
                 "empty": self.filtered_empty,
                 "seeking_work": self.filtered_seeking_work,
                 "not_contractor_friendly": self.filtered_not_contractor_friendly,
+                "irrelevant_role": self.filtered_irrelevant_role,
             },
         }
 
@@ -998,6 +1048,7 @@ def __init__(
         self.include_latest = include_latest
         self.include_non_contractor = include_non_contractor
         self._thread_reports: list[HackerNewsThreadScrapeReport] = []
+        self.irrelevant_external_ids: set[str] = set()
 
     def discover_threads(self) -> list[HackerNewsThread]:
         if self.story_id is not None:
@@ -1016,6 +1067,7 @@ def discover_threads(self) -> list[HackerNewsThread]:
     def collect(self) -> list[JobLeadInput]:
         leads: list[JobLeadInput] = []
         self._thread_reports = []
+        self.irrelevant_external_ids = set()
         for thread in self.discover_threads():
             tree = self.client.get_algolia_item_tree(thread.story_id)
             children = tree.get("children") or []
@@ -1043,6 +1095,12 @@ def collect(self) -> list[JobLeadInput]:
                 if lead is None:
                     report.filtered_not_contractor_friendly += 1
                     continue
+                if not _lead_is_coop_role(
+                    self.source_key, lead.title, lead.body_normalized
+                ):
+                    report.filtered_irrelevant_role += 1
+                    self.irrelevant_external_ids.add(lead.external_id)
+                    continue
                 classification = (lead.metadata or {}).get(
                     "contractor_classification", {}
                 )
@@ -1065,6 +1123,9 @@ def collection_report(self) -> dict[str, Any]:
                 report.filtered_not_contractor_friendly
                 for report in self._thread_reports
             ),
+            "irrelevant_role": sum(
+                report.filtered_irrelevant_role for report in self._thread_reports
+            ),
         }
         return {
             "thread_found": bool(self._thread_reports),
@@ -1090,6 +1151,8 @@ def __init__(
         self.max_pages = max_pages
         self.scanned = 0
         self.expired = 0
+        self.irrelevant_role = 0
+        self.irrelevant_external_ids: set[str] = set()
         self.pages = 0
         self.included = 0
         self.incomplete = False
@@ -1098,7 +1161,12 @@ def __init__(
     def collect(self) -> list[JobLeadInput]:
         leads: list[JobLeadInput] = []
         seen: set[str] = set()
-        self.scanned = self.expired = self.pages = self.included = 0
+        self.scanned = 0
+        self.expired = 0
+        self.irrelevant_role = 0
+        self.irrelevant_external_ids = set()
+        self.pages = 0
+        self.included = 0
         self.incomplete = False
         self.incomplete_reason = None
         now = datetime.now(timezone.utc)
@@ -1162,6 +1230,10 @@ def collect(self) -> list[JobLeadInput]:
                 ):
                     continue
                 seen.add(guid)
+                if not _title_is_coop_role(title):
+                    self.irrelevant_role += 1
+                    self.irrelevant_external_ids.add(guid)
+                    continue
                 body_raw = str(job.get("description") or job.get("excerpt") or "")
                 body = html_to_text(body_raw)
                 posting_type = JobPostingType.PART_TIME
@@ -1226,7 +1298,11 @@ def collection_report(self) -> dict[str, Any]:
             "filtered_out": self.scanned - self.included,
             "filter_reasons": {
                 "expired": self.expired,
-                "invalid_or_duplicate": self.scanned - self.included - self.expired,
+                "irrelevant_role": self.irrelevant_role,
+                "invalid_or_duplicate": self.scanned
+                - self.included
+                - self.expired
+                - self.irrelevant_role,
             },
             "pages_scanned": self.pages,
             "incomplete": self.incomplete,
@@ -1243,11 +1319,14 @@ def __init__(self, *, timeout_seconds: float = 15.0) -> None:
         self.timeout_seconds = timeout_seconds
         self.scanned = 0
         self.included = 0
+        self.irrelevant_role = 0
+        self.irrelevant_external_ids: set[str] = set()
 
     def collect(self) -> list[JobLeadInput]:
         leads: list[JobLeadInput] = []
         seen: set[str] = set()
-        self.scanned = self.included = 0
+        self.scanned = self.included = self.irrelevant_role = 0
+        self.irrelevant_external_ids = set()
         for term in A16Z_SEARCH_TERMS:
             request = Request(
                 f"https://jobs.a16z.com/jobs?{urlencode({'q': term})}",
@@ -1271,6 +1350,10 @@ def collect(self) -> list[JobLeadInput]:
                 path = title_links[0].get("href", "")
                 if not path.startswith("/jobs/") or not _title_is_gig(title):
                     continue
+                if not _title_is_coop_role(title):
+                    self.irrelevant_role += 1
+                    self.irrelevant_external_ids.add(f"https://jobs.a16z.com{path}")
+                    continue
                 source_url = f"https://jobs.a16z.com{path}"
                 if source_url in seen:
                     continue
@@ -1307,7 +1390,12 @@ def collection_report(self) -> dict[str, Any]:
             "potential_gigs_scraped": self.scanned,
             "included": self.included,
             "filtered_out": self.scanned - self.included,
-            "filter_reasons": {"invalid_or_duplicate": self.scanned - self.included},
+            "filter_reasons": {
+                "irrelevant_role": self.irrelevant_role,
+                "invalid_or_duplicate": self.scanned
+                - self.included
+                - self.irrelevant_role,
+            },
             "searches_scanned": len(A16Z_SEARCH_TERMS),
         }
 
@@ -1321,10 +1409,13 @@ def __init__(self, *, timeout_seconds: float = 15.0) -> None:
         self.timeout_seconds = timeout_seconds
         self.scanned = 0
         self.included = 0
+        self.irrelevant_role = 0
+        self.irrelevant_external_ids: set[str] = set()
 
     def collect(self) -> list[JobLeadInput]:
         leads: list[JobLeadInput] = []
-        self.scanned = self.included = 0
+        self.scanned = self.included = self.irrelevant_role = 0
+        self.irrelevant_external_ids = set()
         for company, board in SEQUOIA_GREENHOUSE_BOARDS.items():
             request = Request(
                 f"https://boards-api.greenhouse.io/v1/boards/{board}/jobs",
@@ -1343,6 +1434,11 @@ def collect(self) -> list[JobLeadInput]:
                 source_url = job.get("absolute_url")
                 if not isinstance(title, str) or not _title_is_gig(title):
                     continue
+                if not _title_is_coop_role(title):
+                    self.irrelevant_role += 1
+                    if isinstance(source_url, str):
+                        self.irrelevant_external_ids.add(source_url)
+                    continue
                 if (
                     not isinstance(source_url, str)
                     or urlsplit(source_url).scheme != "https"
@@ -1375,7 +1471,12 @@ def collection_report(self) -> dict[str, Any]:
             "potential_gigs_scraped": self.scanned,
             "included": self.included,
             "filtered_out": self.scanned - self.included,
-            "filter_reasons": {"invalid_or_duplicate": self.scanned - self.included},
+            "filter_reasons": {
+                "irrelevant_role": self.irrelevant_role,
+                "invalid_or_duplicate": self.scanned
+                - self.included
+                - self.irrelevant_role,
+            },
             "portfolio_companies_scanned": len(SEQUOIA_GREENHOUSE_BOARDS),
         }
 
@@ -1468,13 +1569,24 @@ def scrape_job_leads(
             created += 1
         else:
             updated += 1
+    rejected_existing = reject_unreviewed_irrelevant_job_leads(
+        settings,
+        source_key=adapter.source_key,
+        lead_is_relevant=lambda title, body: _lead_is_coop_role(
+            adapter.source_key, title, body
+        ),
+        current_irrelevant_external_ids=getattr(
+            adapter, "irrelevant_external_ids", set()
+        ),
+    )
     return {
         "source": adapter.source_key,
         **collection_report,
         "created": created,
         "updated": updated,
         "total": len(lead_ids),
         "persisted": len(lead_ids),
+        "rejected_existing": rejected_existing,
         "lead_ids": lead_ids,
     }