336 lines
12 KiB
Python
336 lines
12 KiB
Python
"""Batch orchestration: dedup, per-record isolation, cascade, persistence, summary."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass
|
|
|
|
from .careers.cascade import find_careers_page
|
|
from .config import get_settings
|
|
from .db import Database
|
|
from .extract import extract_open_position
|
|
from .http import build_client
|
|
from .models import JobResult, JobStatus, RawJob
|
|
from .resolve import resolve_website
|
|
from .sources.base import JobSource
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public summary result
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class BatchSummary:
|
|
"""Per-stage counts returned (and printed) by run_batch()."""
|
|
|
|
fetched: int = 0
|
|
already_seen: int = 0
|
|
new: int = 0
|
|
website_resolved: int = 0
|
|
careers_found: int = 0
|
|
position_found: int = 0
|
|
needs_review: int = 0
|
|
failed: int = 0
|
|
|
|
@property
|
|
def coverage(self) -> float:
|
|
"""Fraction of new jobs that reached position_found."""
|
|
return self.position_found / self.new if self.new > 0 else 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def run_batch(
|
|
batch_size: int | None = None,
|
|
*,
|
|
search_terms: list[str] | None = None,
|
|
location: str | None = None,
|
|
hours_old: int | None = None,
|
|
db: Database | None = None,
|
|
) -> BatchSummary:
|
|
"""Fetch, dedup, cascade, persist, and export one batch of jobs.
|
|
|
|
Override arguments take precedence over config values; ``None`` means
|
|
"use config default". Returns a :class:`BatchSummary` (also printed).
|
|
|
|
One failing record never aborts the batch — each job is isolated in its
|
|
own try/except; the outer loop continues regardless.
|
|
"""
|
|
settings = get_settings()
|
|
batch_size = batch_size if batch_size is not None else settings.batch_size
|
|
search_terms = search_terms if search_terms is not None else list(settings.search_terms)
|
|
location = location if location is not None else settings.location
|
|
hours_old = hours_old if hours_old is not None else settings.hours_old
|
|
|
|
own_db = db is None
|
|
if own_db:
|
|
db = Database()
|
|
|
|
summary = BatchSummary()
|
|
t0 = time.perf_counter()
|
|
csv_path = None
|
|
|
|
try:
|
|
# ------------------------------------------------------------------
|
|
# Stage 1 — Ingest
|
|
# ------------------------------------------------------------------
|
|
source = _make_source(settings)
|
|
print(f"\n{'='*64}")
|
|
print(
|
|
f" JobSourceAgent batch={batch_size}"
|
|
f" search={search_terms}"
|
|
f" hours_old={hours_old}"
|
|
)
|
|
print(f"{'='*64}")
|
|
print("Stage 1: ingesting jobs…")
|
|
|
|
fetch_limit = min(settings.results_wanted, batch_size)
|
|
raw_jobs: list[RawJob] = source.fetch_recent_jobs(
|
|
search_terms=search_terms,
|
|
location=location,
|
|
hours_old=hours_old,
|
|
results_wanted=fetch_limit,
|
|
)
|
|
summary.fetched = len(raw_jobs)
|
|
|
|
# Dedup: filter out jobs already in the DB; cap new jobs at batch_size.
|
|
new_jobs: list[RawJob] = []
|
|
for raw in raw_jobs:
|
|
if db.is_seen(raw.job_id):
|
|
summary.already_seen += 1
|
|
else:
|
|
new_jobs.append(raw)
|
|
if len(new_jobs) >= batch_size:
|
|
break
|
|
|
|
summary.new = len(new_jobs)
|
|
print(
|
|
f" Fetched {summary.fetched} jobs; "
|
|
f"{summary.already_seen} already seen; "
|
|
f"{summary.new} new to process.\n"
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Stages 1b → 3 — per-record cascade
|
|
# ------------------------------------------------------------------
|
|
if new_jobs:
|
|
with build_client() as client:
|
|
for raw in new_jobs:
|
|
result = JobResult.from_raw(raw)
|
|
print(f"[{raw.job_id}] {raw.company}")
|
|
try:
|
|
_process_one(result, raw, db, client)
|
|
except Exception as exc:
|
|
# Safety net — _process_one should not raise, but guard anyway.
|
|
logger.exception(
|
|
"pipeline: unexpected error for %s (%s): %s",
|
|
raw.job_id, raw.company, exc,
|
|
)
|
|
result.status = JobStatus.failed
|
|
try:
|
|
db.persist_result(result)
|
|
except Exception:
|
|
logger.exception(
|
|
"pipeline: failed to persist error record %s", raw.job_id
|
|
)
|
|
_tally(summary, result)
|
|
print()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Stage 4 — Export CSV
|
|
# ------------------------------------------------------------------
|
|
try:
|
|
csv_path = db.export_csv()
|
|
except Exception as exc:
|
|
logger.error("pipeline: CSV export failed: %s", exc)
|
|
|
|
finally:
|
|
if own_db:
|
|
db.close()
|
|
|
|
elapsed = time.perf_counter() - t0
|
|
_print_summary(summary, elapsed, csv_path)
|
|
return summary
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-record cascade (must not raise — handles all errors internally)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _process_one(
|
|
result: JobResult,
|
|
raw: RawJob,
|
|
db: Database,
|
|
client: object,
|
|
) -> None:
|
|
"""Run stages 1b → 3 for one job and persist the result.
|
|
|
|
Uses early returns for soft misses (needs_review) and catches exceptions
|
|
for hard failures (failed status). Never raises to the caller.
|
|
"""
|
|
# ------------------------------------------------------------------
|
|
# Stage 1b — Resolve company website
|
|
# ------------------------------------------------------------------
|
|
try:
|
|
website = resolve_website(raw.company, raw.website, client=client)
|
|
except Exception as exc:
|
|
logger.warning("pipeline(%s): resolve error: %s", raw.job_id, exc)
|
|
print(f" resolve: ERROR — {exc}")
|
|
result.status = JobStatus.failed
|
|
db.persist_result(result)
|
|
return
|
|
|
|
if not website:
|
|
print(" website: UNRESOLVED")
|
|
result.status = JobStatus.needs_review
|
|
db.persist_result(result)
|
|
return
|
|
|
|
result.website = website
|
|
result.status = JobStatus.website_resolved
|
|
result.company_key = db.company_key_for(raw.company, website)
|
|
print(f" website: {website}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Stage 2 — Find careers page (check company cache first)
|
|
# ------------------------------------------------------------------
|
|
cached_career_url = db.get_cached_career_url(result.company_key)
|
|
|
|
if cached_career_url:
|
|
careers_url = cached_career_url
|
|
careers_method = "cache"
|
|
position_shortcut: str | None = None
|
|
print(f" careers: {careers_url} [cache]")
|
|
else:
|
|
try:
|
|
cr = find_careers_page(website, company_name=raw.company, client=client)
|
|
except Exception as exc:
|
|
logger.warning("pipeline(%s): cascade error: %s", raw.job_id, exc)
|
|
print(f" careers: ERROR — {exc}")
|
|
result.status = JobStatus.needs_review
|
|
db.persist_result(result)
|
|
return
|
|
|
|
careers_url = cr.careers_url
|
|
careers_method = cr.method
|
|
position_shortcut = cr.position_url
|
|
|
|
if not careers_url:
|
|
print(f" careers: MISSED (method={cr.method})")
|
|
result.status = JobStatus.needs_review
|
|
db.persist_result(result)
|
|
return
|
|
|
|
print(
|
|
f" careers: {careers_url}"
|
|
f" [{careers_method} conf={cr.confidence:.2f}]"
|
|
)
|
|
|
|
result.career_page_url = careers_url
|
|
result.careers_method = careers_method
|
|
result.status = JobStatus.careers_found
|
|
|
|
# ------------------------------------------------------------------
|
|
# Stage 3 — Extract one open position
|
|
# ------------------------------------------------------------------
|
|
if position_shortcut:
|
|
# Free shortcut from ATS or browser-agent tier — no extra HTTP call.
|
|
position_url = position_shortcut
|
|
position_method = careers_method
|
|
else:
|
|
try:
|
|
position_url, position_method = extract_open_position(
|
|
careers_url, client=client
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("pipeline(%s): extract error: %s", raw.job_id, exc)
|
|
print(f" position: ERROR — {exc}")
|
|
result.status = JobStatus.needs_review
|
|
db.persist_result(result)
|
|
return
|
|
|
|
if position_url:
|
|
result.open_position_url = position_url
|
|
result.position_method = position_method
|
|
result.status = JobStatus.position_found
|
|
print(f" position: {position_url} [{position_method}]")
|
|
else:
|
|
print(f" position: MISSED (method={position_method})")
|
|
result.status = JobStatus.needs_review
|
|
|
|
db.persist_result(result)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_source(settings) -> JobSource:
|
|
"""Instantiate the configured job source provider."""
|
|
if settings.job_source == "apify":
|
|
from .sources.apify_source import ApifySource
|
|
return ApifySource()
|
|
from .sources.jobspy_source import JobSpySource
|
|
return JobSpySource()
|
|
|
|
|
|
def _tally(summary: BatchSummary, result: JobResult) -> None:
|
|
"""Accumulate per-stage counts from a processed JobResult."""
|
|
if result.website:
|
|
summary.website_resolved += 1
|
|
if result.career_page_url:
|
|
summary.careers_found += 1
|
|
if result.open_position_url:
|
|
summary.position_found += 1
|
|
if result.status == JobStatus.needs_review:
|
|
summary.needs_review += 1
|
|
elif result.status == JobStatus.failed:
|
|
summary.failed += 1
|
|
|
|
|
|
def _print_summary(
|
|
summary: BatchSummary, elapsed: float, csv_path: object
|
|
) -> None:
|
|
"""Print the run summary table to stdout."""
|
|
n = summary.new
|
|
wr = summary.website_resolved
|
|
cf = summary.careers_found
|
|
|
|
print(f"{'='*64}")
|
|
print(f" SUMMARY ({n} new jobs processed in {elapsed:.1f}s)")
|
|
print(f"{'='*64}")
|
|
print(
|
|
f" Stage 1 ingested {summary.fetched:>4}"
|
|
f" ({summary.already_seen} already seen)"
|
|
)
|
|
print(
|
|
f" Stage 1b website resolved {wr:>4} / {n}"
|
|
f" ({wr / max(n, 1) * 100:.0f}%)"
|
|
)
|
|
print(
|
|
f" Stage 2 careers found {cf:>4} / {wr}"
|
|
f" ({cf / max(wr, 1) * 100:.0f}% of resolved)"
|
|
)
|
|
print(
|
|
f" Stage 3 position found {summary.position_found:>4} / {cf}"
|
|
f" ({summary.position_found / max(cf, 1) * 100:.0f}% of careers)"
|
|
)
|
|
print(f" Needs review {summary.needs_review:>4}")
|
|
print(f" Failed {summary.failed:>4}")
|
|
print(
|
|
f" End-to-end coverage "
|
|
f"{summary.coverage * 100:.0f}%"
|
|
f" ({summary.position_found}/{n} new jobs fully resolved)"
|
|
)
|
|
if csv_path:
|
|
print(f" CSV written to {csv_path}")
|
|
print()
|