#!/usr/bin/env python """End-to-end smoke test: Stage 1→3 on a 10-job batch. Usage: .venv/bin/python scripts/e2e_smoke.py """ from __future__ import annotations import logging import sys import time from dataclasses import dataclass, field # Make sure we run from the repo root so imports resolve correctly. sys.path.insert(0, ".") logging.basicConfig( level=logging.WARNING, # keep jobspy/httpx chatter suppressed format="%(name)s: %(message)s", ) # Enable our own modules at INFO so we see cascade method names. for _mod in ("jobsource.resolve", "jobsource.careers.cascade", "jobsource.careers.ats", "jobsource.careers.heuristics", "jobsource.careers.classify_llm", "jobsource.extract"): logging.getLogger(_mod).setLevel(logging.INFO) from jobsource.careers.cascade import find_careers_page from jobsource.config import get_settings from jobsource.extract import extract_open_position from jobsource.http import build_client from jobsource.models import RawJob from jobsource.resolve import resolve_website from jobsource.sources.jobspy_source import JobSpySource # --------------------------------------------------------------------------- # Per-record result # --------------------------------------------------------------------------- @dataclass class Result: job_id: str company: str website: str | None = None careers_url: str | None = None careers_method: str | None = None position_url: str | None = None position_method: str | None = None error: str | None = None # --------------------------------------------------------------------------- # Main runner # --------------------------------------------------------------------------- BATCH_SIZE = 10 SEARCH_TERM = "software engineer" LOCATION = "United States" HOURS_OLD = 72 def run() -> None: settings = get_settings() print(f"\n{'='*64}") print(f" E2E smoke batch={BATCH_SIZE} search={SEARCH_TERM!r} hours_old={HOURS_OLD}") print(f"{'='*64}\n") # ------------------------------------------------------------------ # Stage 1 — Ingest # ------------------------------------------------------------------ print("Stage 1: ingesting…") source = JobSpySource() t0 = time.perf_counter() raw_jobs: list[RawJob] = source.fetch_recent_jobs( search_terms=[SEARCH_TERM], location=LOCATION, hours_old=HOURS_OLD, results_wanted=BATCH_SIZE + 10, # fetch a few extra to cover dupes ) elapsed = time.perf_counter() - t0 raw_jobs = raw_jobs[:BATCH_SIZE] print(f" Ingested {len(raw_jobs)} unique jobs in {elapsed:.1f}s\n") if not raw_jobs: print("No jobs returned — aborting.") return results: list[Result] = [] with build_client() as client: for raw in raw_jobs: res = Result(job_id=raw.job_id, company=raw.company) print(f"[{raw.job_id}] {raw.company}") # ------------------------------------------------------- # Stage 1b — Resolve website # ------------------------------------------------------- try: res.website = resolve_website( raw.company, raw.website, client=client ) except Exception as exc: res.error = f"resolve: {exc}" print(f" resolve ERROR: {exc}") results.append(res) continue if not res.website: print(f" website: UNRESOLVED") results.append(res) continue print(f" website: {res.website}") # ------------------------------------------------------- # Stage 2 — Find careers page # ------------------------------------------------------- try: cr = find_careers_page( res.website, company_name=raw.company, client=client, ) res.careers_url = cr.careers_url res.careers_method = cr.method except Exception as exc: res.error = f"cascade: {exc}" print(f" careers ERROR: {exc}") results.append(res) continue if not res.careers_url: print(f" careers: MISSED (method={cr.method})") results.append(res) continue print(f" careers: {res.careers_url} [{cr.method} conf={cr.confidence:.2f}]") # ------------------------------------------------------- # Stage 3 — Extract one open position # ------------------------------------------------------- try: pos_url, pos_method = extract_open_position( res.careers_url, client=client ) res.position_url = pos_url res.position_method = pos_method except Exception as exc: res.error = f"extract: {exc}" print(f" position ERROR: {exc}") results.append(res) continue if res.position_url: print(f" position: {res.position_url} [{pos_method}]") else: print(f" position: MISSED (method={pos_method})") results.append(res) print() # ------------------------------------------------------------------ # Summary # ------------------------------------------------------------------ n_total = len(results) n_website = sum(1 for r in results if r.website) n_careers = sum(1 for r in results if r.careers_url) n_position = sum(1 for r in results if r.position_url) n_errors = sum(1 for r in results if r.error) print(f"\n{'='*64}") print(f" SUMMARY ({n_total} jobs processed)") print(f"{'='*64}") print(f" Stage 1 ingested {n_total:>3}") print(f" Stage 1b website resolved {n_website:>3} / {n_total} ({n_website/n_total*100:.0f}%)") print(f" Stage 2 careers found {n_careers:>3} / {n_website} ({n_careers/max(n_website,1)*100:.0f}% of resolved)") print(f" Stage 3 position found {n_position:>3} / {n_careers} ({n_position/max(n_careers,1)*100:.0f}% of careers)") print(f" Errors {n_errors:>3}") # careers method breakdown if n_careers: method_counts: dict[str, int] = {} for r in results: if r.careers_method and r.careers_url: method_counts[r.careers_method] = method_counts.get(r.careers_method, 0) + 1 print(f"\n Careers method breakdown:") for m, c in sorted(method_counts.items(), key=lambda x: -x[1]): print(f" {m:<30} {c}") # position method breakdown if n_position: pos_method_counts: dict[str, int] = {} for r in results: if r.position_method and r.position_url: pos_method_counts[r.position_method] = pos_method_counts.get(r.position_method, 0) + 1 print(f"\n Position method breakdown:") for m, c in sorted(pos_method_counts.items(), key=lambda x: -x[1]): print(f" {m:<30} {c}") # CSV-ready output print(f"\n{'='*64}") print(" company_name, career_page_url, open_position_url") print(f"{'='*64}") for r in results: print(f" {r.company!r}, {r.careers_url or ''!r}, {r.position_url or ''!r}") print() if __name__ == "__main__": run()