first implementation
This commit is contained in:
159
validate.py
Normal file
159
validate.py
Normal file
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python
|
||||
"""Live validation against the CLAUDE.md success criteria.
|
||||
|
||||
Runs a small batch through the full pipeline, then immediately re-runs to
|
||||
verify dedup. Prints PASS/FAIL for each criterion and exits non-zero if
|
||||
any check fails.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python validate.py
|
||||
.venv/bin/python validate.py --batch-size 10 --search "data engineer"
|
||||
.venv/bin/python validate.py --help
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from jobsource.db import Database
|
||||
from jobsource.models import CSV_COLUMNS
|
||||
from jobsource.pipeline import run_batch
|
||||
|
||||
|
||||
def _check(label: str, passed: bool, detail: str = "") -> bool:
|
||||
mark = "PASS" if passed else "FAIL"
|
||||
suffix = f" — {detail}" if detail else ""
|
||||
print(f" [{mark}] {label}{suffix}")
|
||||
return passed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--batch-size", type=int, default=5,
|
||||
help="Number of new jobs to process (default: 5)")
|
||||
parser.add_argument("--search", default="software engineer",
|
||||
help="Search term (default: 'software engineer')")
|
||||
parser.add_argument("--location", default="United States",
|
||||
help="Location filter (default: 'United States')")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"\n{'='*64}")
|
||||
print(f" validate.py batch={args.batch_size} search={args.search!r}")
|
||||
print(f"{'='*64}\n")
|
||||
|
||||
checks: list[bool] = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "validate.db"
|
||||
csv_path = Path(tmp) / "results.csv"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Run 1: initial batch
|
||||
# ------------------------------------------------------------------
|
||||
print("Run 1 (initial batch)…")
|
||||
s1 = exc1 = None
|
||||
try:
|
||||
with Database(path=db_path) as db:
|
||||
s1 = run_batch(
|
||||
batch_size=args.batch_size,
|
||||
search_terms=[args.search],
|
||||
location=args.location,
|
||||
db=db,
|
||||
)
|
||||
db.export_csv(path=csv_path)
|
||||
except Exception as exc:
|
||||
exc1 = exc
|
||||
|
||||
# [1] Batch completes without unhandled exception
|
||||
checks.append(_check(
|
||||
"Batch completes without unhandled exception",
|
||||
exc1 is None,
|
||||
repr(exc1) if exc1 else "",
|
||||
))
|
||||
if exc1 is not None:
|
||||
print("\nFatal error in run 1 — cannot continue.")
|
||||
return 1
|
||||
|
||||
# [2] CSV has exactly the three contract columns
|
||||
ok_csv = False
|
||||
csv_detail = "not written"
|
||||
if csv_path.exists():
|
||||
with csv_path.open() as f:
|
||||
reader = csv.DictReader(f)
|
||||
rows = list(reader)
|
||||
actual = tuple(reader.fieldnames or [])
|
||||
ok_csv = actual == CSV_COLUMNS
|
||||
csv_detail = (
|
||||
f"columns={actual}" if not ok_csv
|
||||
else f"{len(rows)} data rows columns={actual}"
|
||||
)
|
||||
checks.append(_check("CSV has exactly the three contract columns", ok_csv, csv_detail))
|
||||
|
||||
# [3] Per-stage summary with coverage metric
|
||||
ok_summary = s1 is not None
|
||||
checks.append(_check(
|
||||
"Per-stage summary returned",
|
||||
ok_summary,
|
||||
(f"coverage={s1.coverage*100:.0f}% "
|
||||
f"({s1.position_found}/{s1.new} new jobs fully resolved)") if s1 else "",
|
||||
))
|
||||
|
||||
# warn if no jobs were fetched (network/API issue, not a hard failure)
|
||||
if s1 and s1.new == 0:
|
||||
print(" NOTE: 0 new jobs fetched — dedup check may be vacuously true.")
|
||||
print(" Check your search terms, network, or hours_old setting.\n")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Run 2: dedup check
|
||||
# ------------------------------------------------------------------
|
||||
print("Run 2 (dedup check — same DB)…")
|
||||
s2 = exc2 = None
|
||||
try:
|
||||
with Database(path=db_path) as db2:
|
||||
s2 = run_batch(
|
||||
batch_size=args.batch_size,
|
||||
search_terms=[args.search],
|
||||
location=args.location,
|
||||
db=db2,
|
||||
)
|
||||
except Exception as exc:
|
||||
exc2 = exc
|
||||
|
||||
# [4] Re-run processes 0 new jobs
|
||||
ok_dedup = exc2 is None and s2 is not None and s2.new == 0
|
||||
checks.append(_check(
|
||||
"Re-run processes 0 new jobs (dedup proven)",
|
||||
ok_dedup,
|
||||
(f"new={s2.new if s2 else 'N/A'}, seen={s2.already_seen if s2 else 'N/A'}"
|
||||
if not ok_dedup
|
||||
else f"0 new / {s2.already_seen} already seen"),
|
||||
))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Spot-check reminder (manual step — not auto-checkable)
|
||||
# ------------------------------------------------------------------
|
||||
if s1 and s1.position_found > 0:
|
||||
print(
|
||||
f"\n Manual spot-check: open output/results.csv and verify that\n"
|
||||
f" career_page_url and open_position_url return HTTP 200 in a browser."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Verdict
|
||||
# ------------------------------------------------------------------
|
||||
n_pass = sum(checks)
|
||||
n_total = len(checks)
|
||||
print(f"\n{'='*64}")
|
||||
overall = all(checks)
|
||||
print(f" Overall: {'PASS' if overall else 'FAIL'} ({n_pass}/{n_total} automated checks passed)")
|
||||
print(f"{'='*64}\n")
|
||||
return 0 if overall else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user