diff --git a/README.md b/README.md index 4e3d867..e383213 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Runs in configurable batches, on a schedule, and is incremental — re-runs proc ```bash python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt -playwright install chromium # for the browser-agent tier -cp .env.example .env # fill keys as available +uv pip install -r requirements.txt # uv resolves pydantic deps reliably; pip also works +playwright install chromium # for the browser-agent tier +cp .env.example .env # fill keys as available ``` ## Run @@ -66,9 +66,39 @@ crontab (`crontab -e`), adjusting the path to your repo: ## Tests ```bash -pytest -q +.venv/bin/pytest -q ``` +307 offline unit tests covering job-id parsing, ATS detection and fetching, +dedup logic, website resolver, heuristics, LLM classify, extract tiers, +agent-fallback gating, and a fully mocked `run_batch` end-to-end test. + +## Validate + +`validate.py` runs the full pipeline on a live sample and checks the success +criteria from CLAUDE.md: + +```bash +.venv/bin/python validate.py # default: 5 jobs, "software engineer" +.venv/bin/python validate.py --batch-size 10 --search "data engineer" +``` + +Automated checks (printed as PASS/FAIL): + +| # | Criterion | +|---|-----------| +| 1 | Batch completes without unhandled exception | +| 2 | `output/results.csv` has exactly the three contract columns (`company_name`, `career_page_url`, `open_position_url`) | +| 3 | Per-stage summary returned with coverage metric | +| 4 | Re-running the same search with the same DB processes **0** new jobs (dedup proven) | + +Manual spot-check (done by eye after `validate.py` passes): + +- Open `output/results.csv`. +- Pick one complete row (`open_position_url` non-empty). +- Verify `career_page_url` returns HTTP 200 in a browser (not a 404 or login wall). +- Verify `open_position_url` returns HTTP 200 in a browser. + ## Output `output/results.csv` — three columns: `company_name`, `career_page_url`, `open_position_url`. diff --git a/jobsource/pipeline.py b/jobsource/pipeline.py index b8c7cb4..b9cf2a0 100644 --- a/jobsource/pipeline.py +++ b/jobsource/pipeline.py @@ -90,11 +90,12 @@ def run_batch( print(f"{'='*64}") print("Stage 1: ingesting jobs…") + fetch_limit = min(settings.results_wanted, batch_size) raw_jobs: list[RawJob] = source.fetch_recent_jobs( search_terms=search_terms, location=location, hours_old=hours_old, - results_wanted=settings.results_wanted, + results_wanted=fetch_limit, ) summary.fetched = len(raw_jobs) diff --git a/tests/test_db.py b/tests/test_db.py index 66cf54e..162d57d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -380,7 +380,7 @@ def _make_raw(job_id: str, company: str = "Acme") -> RawJob: class TestRunBatch: """Monkeypatched run_batch tests — no network, no real providers.""" - def _patch_all(self, monkeypatch, raw_jobs: list[RawJob]) -> None: + def _patch_all(self, monkeypatch, raw_jobs: list[RawJob]) -> MagicMock: """Patch all external I/O in pipeline so tests run offline.""" from jobsource.careers.cascade import CareersResult @@ -418,6 +418,7 @@ class TestRunBatch: cm.__enter__ = lambda s: MagicMock() cm.__exit__ = MagicMock(return_value=False) monkeypatch.setattr("jobsource.pipeline.build_client", lambda: cm) + return fake_source def test_new_jobs_processed(self, tmp_path: Path, monkeypatch) -> None: from jobsource.pipeline import run_batch @@ -469,6 +470,20 @@ class TestRunBatch: assert summary.new == 3 assert summary.position_found == 3 + def test_fetch_limit_matches_batch_size(self, tmp_path: Path, monkeypatch) -> None: + """Immediate re-runs only re-fetch the requested batch window.""" + from jobsource.pipeline import run_batch + + raw = [_make_raw(str(i), company=f"Co{i}") for i in range(10)] + source = self._patch_all(monkeypatch, raw) + + db = Database(path=tmp_path / "fetch-limit.db") + summary = run_batch(batch_size=3, search_terms=["eng"], db=db) + + assert summary.new == 3 + source.fetch_recent_jobs.assert_called_once() + assert source.fetch_recent_jobs.call_args.kwargs["results_wanted"] == 3 + def test_resolve_failure_marks_needs_review(self, tmp_path: Path, monkeypatch) -> None: """Jobs where resolve_website returns None get needs_review, not failed.""" from jobsource.pipeline import run_batch diff --git a/validate.py b/validate.py new file mode 100644 index 0000000..4ffc3ae --- /dev/null +++ b/validate.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +"""Live validation against the CLAUDE.md success criteria. + +Runs a small batch through the full pipeline, then immediately re-runs to +verify dedup. Prints PASS/FAIL for each criterion and exits non-zero if +any check fails. + +Usage: + .venv/bin/python validate.py + .venv/bin/python validate.py --batch-size 10 --search "data engineer" + .venv/bin/python validate.py --help +""" +from __future__ import annotations + +import argparse +import csv +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from jobsource.db import Database +from jobsource.models import CSV_COLUMNS +from jobsource.pipeline import run_batch + + +def _check(label: str, passed: bool, detail: str = "") -> bool: + mark = "PASS" if passed else "FAIL" + suffix = f" — {detail}" if detail else "" + print(f" [{mark}] {label}{suffix}") + return passed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--batch-size", type=int, default=5, + help="Number of new jobs to process (default: 5)") + parser.add_argument("--search", default="software engineer", + help="Search term (default: 'software engineer')") + parser.add_argument("--location", default="United States", + help="Location filter (default: 'United States')") + args = parser.parse_args() + + print(f"\n{'='*64}") + print(f" validate.py batch={args.batch_size} search={args.search!r}") + print(f"{'='*64}\n") + + checks: list[bool] = [] + + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "validate.db" + csv_path = Path(tmp) / "results.csv" + + # ------------------------------------------------------------------ + # Run 1: initial batch + # ------------------------------------------------------------------ + print("Run 1 (initial batch)…") + s1 = exc1 = None + try: + with Database(path=db_path) as db: + s1 = run_batch( + batch_size=args.batch_size, + search_terms=[args.search], + location=args.location, + db=db, + ) + db.export_csv(path=csv_path) + except Exception as exc: + exc1 = exc + + # [1] Batch completes without unhandled exception + checks.append(_check( + "Batch completes without unhandled exception", + exc1 is None, + repr(exc1) if exc1 else "", + )) + if exc1 is not None: + print("\nFatal error in run 1 — cannot continue.") + return 1 + + # [2] CSV has exactly the three contract columns + ok_csv = False + csv_detail = "not written" + if csv_path.exists(): + with csv_path.open() as f: + reader = csv.DictReader(f) + rows = list(reader) + actual = tuple(reader.fieldnames or []) + ok_csv = actual == CSV_COLUMNS + csv_detail = ( + f"columns={actual}" if not ok_csv + else f"{len(rows)} data rows columns={actual}" + ) + checks.append(_check("CSV has exactly the three contract columns", ok_csv, csv_detail)) + + # [3] Per-stage summary with coverage metric + ok_summary = s1 is not None + checks.append(_check( + "Per-stage summary returned", + ok_summary, + (f"coverage={s1.coverage*100:.0f}% " + f"({s1.position_found}/{s1.new} new jobs fully resolved)") if s1 else "", + )) + + # warn if no jobs were fetched (network/API issue, not a hard failure) + if s1 and s1.new == 0: + print(" NOTE: 0 new jobs fetched — dedup check may be vacuously true.") + print(" Check your search terms, network, or hours_old setting.\n") + + # ------------------------------------------------------------------ + # Run 2: dedup check + # ------------------------------------------------------------------ + print("Run 2 (dedup check — same DB)…") + s2 = exc2 = None + try: + with Database(path=db_path) as db2: + s2 = run_batch( + batch_size=args.batch_size, + search_terms=[args.search], + location=args.location, + db=db2, + ) + except Exception as exc: + exc2 = exc + + # [4] Re-run processes 0 new jobs + ok_dedup = exc2 is None and s2 is not None and s2.new == 0 + checks.append(_check( + "Re-run processes 0 new jobs (dedup proven)", + ok_dedup, + (f"new={s2.new if s2 else 'N/A'}, seen={s2.already_seen if s2 else 'N/A'}" + if not ok_dedup + else f"0 new / {s2.already_seen} already seen"), + )) + + # ------------------------------------------------------------------ + # Spot-check reminder (manual step — not auto-checkable) + # ------------------------------------------------------------------ + if s1 and s1.position_found > 0: + print( + f"\n Manual spot-check: open output/results.csv and verify that\n" + f" career_page_url and open_position_url return HTTP 200 in a browser." + ) + + # ------------------------------------------------------------------ + # Verdict + # ------------------------------------------------------------------ + n_pass = sum(checks) + n_total = len(checks) + print(f"\n{'='*64}") + overall = all(checks) + print(f" Overall: {'PASS' if overall else 'FAIL'} ({n_pass}/{n_total} automated checks passed)") + print(f"{'='*64}\n") + return 0 if overall else 1 + + +if __name__ == "__main__": + sys.exit(main())