JobSourceAgent/validate.py

#!/usr/bin/env python
"""Live validation against the CLAUDE.md success criteria.

Runs a small batch through the full pipeline, then immediately re-runs to
verify dedup.  Prints PASS/FAIL for each criterion and exits non-zero if
any check fails.

Usage:
    .venv/bin/python validate.py
    .venv/bin/python validate.py --batch-size 10 --search "data engineer"
    .venv/bin/python validate.py --help
"""
from __future__ import annotations

import argparse
import csv
import sys
import tempfile
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))

from jobsource.db import Database
from jobsource.models import CSV_COLUMNS
from jobsource.pipeline import run_batch


def _check(label: str, passed: bool, detail: str = "") -> bool:
    mark = "PASS" if passed else "FAIL"
    suffix = f"  — {detail}" if detail else ""
    print(f"  [{mark}] {label}{suffix}")
    return passed


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--batch-size", type=int, default=5,
                        help="Number of new jobs to process (default: 5)")
    parser.add_argument("--search", default="software engineer",
                        help="Search term (default: 'software engineer')")
    parser.add_argument("--location", default="United States",
                        help="Location filter (default: 'United States')")
    args = parser.parse_args()

    print(f"\n{'='*64}")
    print(f"  validate.py  batch={args.batch_size}  search={args.search!r}")
    print(f"{'='*64}\n")

    checks: list[bool] = []

    with tempfile.TemporaryDirectory() as tmp:
        db_path  = Path(tmp) / "validate.db"
        csv_path = Path(tmp) / "results.csv"

        # ------------------------------------------------------------------
        # Run 1: initial batch
        # ------------------------------------------------------------------
        print("Run 1 (initial batch)…")
        s1 = exc1 = None
        try:
            with Database(path=db_path) as db:
                s1 = run_batch(
                    batch_size=args.batch_size,
                    search_terms=[args.search],
                    location=args.location,
                    db=db,
                )
                db.export_csv(path=csv_path)
        except Exception as exc:
            exc1 = exc

        # [1] Batch completes without unhandled exception
        checks.append(_check(
            "Batch completes without unhandled exception",
            exc1 is None,
            repr(exc1) if exc1 else "",
        ))
        if exc1 is not None:
            print("\nFatal error in run 1 — cannot continue.")
            return 1

        # [2] CSV has exactly the three contract columns
        ok_csv    = False
        csv_detail = "not written"
        if csv_path.exists():
            with csv_path.open() as f:
                reader = csv.DictReader(f)
                rows   = list(reader)
                actual = tuple(reader.fieldnames or [])
            ok_csv     = actual == CSV_COLUMNS
            csv_detail = (
                f"columns={actual}" if not ok_csv
                else f"{len(rows)} data rows  columns={actual}"
            )
        checks.append(_check("CSV has exactly the three contract columns", ok_csv, csv_detail))

        # [3] Per-stage summary with coverage metric
        ok_summary = s1 is not None
        checks.append(_check(
            "Per-stage summary returned",
            ok_summary,
            (f"coverage={s1.coverage*100:.0f}%  "
             f"({s1.position_found}/{s1.new} new jobs fully resolved)") if s1 else "",
        ))

        # warn if no jobs were fetched (network/API issue, not a hard failure)
        if s1 and s1.new == 0:
            print("  NOTE: 0 new jobs fetched — dedup check may be vacuously true.")
            print("        Check your search terms, network, or hours_old setting.\n")

        # ------------------------------------------------------------------
        # Run 2: dedup check
        # ------------------------------------------------------------------
        print("Run 2 (dedup check — same DB)…")
        s2 = exc2 = None
        try:
            with Database(path=db_path) as db2:
                s2 = run_batch(
                    batch_size=args.batch_size,
                    search_terms=[args.search],
                    location=args.location,
                    db=db2,
                )
        except Exception as exc:
            exc2 = exc

        # [4] Re-run processes 0 new jobs
        ok_dedup = exc2 is None and s2 is not None and s2.new == 0
        checks.append(_check(
            "Re-run processes 0 new jobs (dedup proven)",
            ok_dedup,
            (f"new={s2.new if s2 else 'N/A'}, seen={s2.already_seen if s2 else 'N/A'}"
             if not ok_dedup
             else f"0 new / {s2.already_seen} already seen"),
        ))

    # ------------------------------------------------------------------
    # Spot-check reminder (manual step — not auto-checkable)
    # ------------------------------------------------------------------
    if s1 and s1.position_found > 0:
        print(
            f"\n  Manual spot-check: open output/results.csv and verify that\n"
            f"  career_page_url and open_position_url return HTTP 200 in a browser."
        )

    # ------------------------------------------------------------------
    # Verdict
    # ------------------------------------------------------------------
    n_pass = sum(checks)
    n_total = len(checks)
    print(f"\n{'='*64}")
    overall = all(checks)
    print(f"  Overall: {'PASS' if overall else 'FAIL'}  ({n_pass}/{n_total} automated checks passed)")
    print(f"{'='*64}\n")
    return 0 if overall else 1


if __name__ == "__main__":
    sys.exit(main())