551 lines
21 KiB
Python
551 lines
21 KiB
Python
"""Tests for jobsource/db.py and jobsource/pipeline.py.
|
|
|
|
All tests are offline — no network calls, no real job source.
|
|
The DB uses tmp_path (file-backed) or ":memory:" via path override.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from jobsource.db import Database, _company_key
|
|
from jobsource.models import CSV_COLUMNS, JobResult, JobStatus, RawJob
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _db(tmp_path: Path) -> Database:
|
|
"""Open a fresh file-backed Database in tmp_path."""
|
|
return Database(path=tmp_path / "test.db")
|
|
|
|
|
|
def _result(
|
|
job_id: str = "1",
|
|
company_name: str = "Acme",
|
|
website: str | None = "https://acme.com",
|
|
career_url: str | None = "https://acme.com/careers",
|
|
position_url: str | None = "https://acme.com/careers/jobs/42",
|
|
status: JobStatus = JobStatus.position_found,
|
|
company_key: str | None = None,
|
|
) -> JobResult:
|
|
return JobResult(
|
|
job_id=job_id,
|
|
company_name=company_name,
|
|
company_key=company_key or _company_key(company_name, website),
|
|
website=website,
|
|
career_page_url=career_url,
|
|
open_position_url=position_url,
|
|
status=status,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _company_key (pure function)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompanyKey:
|
|
def test_domain_stripped_of_www(self) -> None:
|
|
assert _company_key("Acme", "https://www.acme.com/about") == "acme.com"
|
|
|
|
def test_domain_lowercase(self) -> None:
|
|
assert _company_key("Foo", "https://Foo.IO") == "foo.io"
|
|
|
|
def test_no_website_uses_name(self) -> None:
|
|
assert _company_key("Globex Corp", None) == "globex corp"
|
|
|
|
def test_no_website_lowercased(self) -> None:
|
|
assert _company_key("Initech LLC", None) == "initech llc"
|
|
|
|
def test_empty_host_falls_back_to_name(self) -> None:
|
|
# Malformed URL with empty hostname
|
|
result = _company_key("BadCo", "://no-host")
|
|
assert result == "badco"
|
|
|
|
def test_empty_name_and_no_website_returns_unknown(self) -> None:
|
|
assert _company_key("", None) == "unknown"
|
|
|
|
def test_subpath_ignored(self) -> None:
|
|
assert _company_key("X", "https://x.example.com/jobs/42") == "x.example.com"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Database.init_schema / open
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestDatabaseSchema:
|
|
def test_creates_tables(self, tmp_path: Path) -> None:
|
|
db = _db(tmp_path)
|
|
try:
|
|
cur = db._conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
|
)
|
|
tables = {r[0] for r in cur.fetchall()}
|
|
assert "companies" in tables
|
|
assert "jobs" in tables
|
|
finally:
|
|
db.close()
|
|
|
|
def test_idempotent_schema(self, tmp_path: Path) -> None:
|
|
"""Opening the same DB twice must not raise."""
|
|
db1 = Database(path=tmp_path / "idempotent.db")
|
|
db1.close()
|
|
db2 = Database(path=tmp_path / "idempotent.db")
|
|
db2.close()
|
|
|
|
def test_parent_dirs_created(self, tmp_path: Path) -> None:
|
|
nested = tmp_path / "a" / "b" / "c" / "test.db"
|
|
db = Database(path=nested)
|
|
try:
|
|
assert nested.exists()
|
|
finally:
|
|
db.close()
|
|
|
|
def test_context_manager(self, tmp_path: Path) -> None:
|
|
with Database(path=tmp_path / "cm.db") as db:
|
|
assert not db.is_seen("x")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# is_seen / dedup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestIsSeeen:
|
|
def test_unseen_job_returns_false(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
assert not db.is_seen("99999")
|
|
|
|
def test_seen_after_persist(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="42")
|
|
db.persist_result(r)
|
|
assert db.is_seen("42")
|
|
|
|
def test_different_job_ids_independent(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.persist_result(_result(job_id="1"))
|
|
assert db.is_seen("1")
|
|
assert not db.is_seen("2")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# company_key_for
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompanyKeyFor:
|
|
def test_delegates_to_pure_fn(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
assert db.company_key_for("Acme", "https://acme.com") == "acme.com"
|
|
assert db.company_key_for("Foo LLC", None) == "foo llc"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# get_cached_career_url
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCachedCareerUrl:
|
|
def test_miss_returns_none(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
assert db.get_cached_career_url("acme.com") is None
|
|
|
|
def test_hit_after_persist(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="1", website="https://acme.com", career_url="https://acme.com/careers")
|
|
db.persist_result(r)
|
|
assert db.get_cached_career_url("acme.com") == "https://acme.com/careers"
|
|
|
|
def test_null_career_url_not_cached(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="1", career_url=None, status=JobStatus.needs_review)
|
|
db.persist_result(r)
|
|
assert db.get_cached_career_url("acme.com") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# upsert_company COALESCE / first_seen preservation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestUpsertCompany:
|
|
def test_insert_then_null_fill(self, tmp_path: Path) -> None:
|
|
"""A NULL value in the second upsert fills the existing NULL without error."""
|
|
with _db(tmp_path) as db:
|
|
db.upsert_company("x.com", "X", None, None)
|
|
db.upsert_company("x.com", "X", "https://x.com", "https://x.com/careers")
|
|
row = db._conn.execute(
|
|
"SELECT website, career_url FROM companies WHERE company_key='x.com'"
|
|
).fetchone()
|
|
assert row["website"] == "https://x.com"
|
|
assert row["career_url"] == "https://x.com/careers"
|
|
|
|
def test_existing_value_not_clobbered(self, tmp_path: Path) -> None:
|
|
"""A second upsert with NULL must not overwrite an existing value."""
|
|
with _db(tmp_path) as db:
|
|
db.upsert_company("y.com", "Y", "https://y.com", "https://y.com/jobs")
|
|
db.upsert_company("y.com", "Y", None, None)
|
|
row = db._conn.execute(
|
|
"SELECT website, career_url FROM companies WHERE company_key='y.com'"
|
|
).fetchone()
|
|
assert row["website"] == "https://y.com"
|
|
assert row["career_url"] == "https://y.com/jobs"
|
|
|
|
def test_first_seen_preserved(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.upsert_company("z.com", "Z", None, None)
|
|
row1 = db._conn.execute(
|
|
"SELECT first_seen FROM companies WHERE company_key='z.com'"
|
|
).fetchone()
|
|
db.upsert_company("z.com", "Z", "https://z.com", "https://z.com/careers")
|
|
row2 = db._conn.execute(
|
|
"SELECT first_seen FROM companies WHERE company_key='z.com'"
|
|
).fetchone()
|
|
assert row1["first_seen"] == row2["first_seen"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# persist_result / upsert_job
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPersistResult:
|
|
def test_both_tables_populated(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="10", company_name="Initech", website="https://initech.com")
|
|
db.persist_result(r)
|
|
job = db._conn.execute(
|
|
"SELECT * FROM jobs WHERE job_id='10'"
|
|
).fetchone()
|
|
assert job is not None
|
|
assert job["status"] == "position_found"
|
|
company = db._conn.execute(
|
|
"SELECT * FROM companies WHERE company_key='initech.com'"
|
|
).fetchone()
|
|
assert company is not None
|
|
assert company["name"] == "Initech"
|
|
|
|
def test_listed_at_null_stored_as_null(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="20")
|
|
assert r.listed_at is None
|
|
db.persist_result(r)
|
|
row = db._conn.execute(
|
|
"SELECT listed_at FROM jobs WHERE job_id='20'"
|
|
).fetchone()
|
|
assert row["listed_at"] is None
|
|
|
|
def test_listed_at_serialized(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="30")
|
|
r.listed_at = datetime(2026, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
|
|
db.persist_result(r)
|
|
row = db._conn.execute(
|
|
"SELECT listed_at FROM jobs WHERE job_id='30'"
|
|
).fetchone()
|
|
assert row["listed_at"] is not None
|
|
assert "2026" in row["listed_at"]
|
|
|
|
def test_upsert_fills_null_position(self, tmp_path: Path) -> None:
|
|
"""A second persist with position_url fills a prior NULL."""
|
|
with _db(tmp_path) as db:
|
|
r1 = _result(job_id="50", position_url=None, status=JobStatus.needs_review)
|
|
db.persist_result(r1)
|
|
r2 = _result(job_id="50", position_url="https://acme.com/jobs/1", status=JobStatus.position_found)
|
|
db.persist_result(r2)
|
|
row = db._conn.execute(
|
|
"SELECT position_url, status FROM jobs WHERE job_id='50'"
|
|
).fetchone()
|
|
assert row["position_url"] == "https://acme.com/jobs/1"
|
|
assert row["status"] == "position_found"
|
|
|
|
def test_job_first_seen_preserved(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="60", position_url=None, status=JobStatus.needs_review)
|
|
db.persist_result(r)
|
|
row1 = db._conn.execute(
|
|
"SELECT first_seen FROM jobs WHERE job_id='60'"
|
|
).fetchone()
|
|
r2 = _result(job_id="60", position_url="https://acme.com/jobs/1", status=JobStatus.position_found)
|
|
db.persist_result(r2)
|
|
row2 = db._conn.execute(
|
|
"SELECT first_seen FROM jobs WHERE job_id='60'"
|
|
).fetchone()
|
|
assert row1["first_seen"] == row2["first_seen"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# counts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCounts:
|
|
def test_empty_db(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
assert db.counts() == {}
|
|
|
|
def test_counts_grouped_by_status(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.persist_result(_result(job_id="1", status=JobStatus.position_found))
|
|
db.persist_result(_result(job_id="2", position_url=None, status=JobStatus.needs_review))
|
|
db.persist_result(_result(job_id="3", position_url=None, status=JobStatus.needs_review))
|
|
c = db.counts()
|
|
assert c["position_found"] == 1
|
|
assert c["needs_review"] == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# export_csv
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestExportCsv:
|
|
def test_header_matches_csv_columns(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.persist_result(_result(job_id="1"))
|
|
out = tmp_path / "out.csv"
|
|
db.export_csv(path=out)
|
|
with out.open() as f:
|
|
reader = csv.DictReader(f)
|
|
assert tuple(reader.fieldnames or []) == CSV_COLUMNS
|
|
|
|
def test_null_fields_become_empty_string(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
r = _result(job_id="1", career_url=None, position_url=None, status=JobStatus.needs_review)
|
|
db.persist_result(r)
|
|
out = tmp_path / "out.csv"
|
|
db.export_csv(path=out)
|
|
with out.open() as f:
|
|
rows = list(csv.DictReader(f))
|
|
assert rows[0]["career_page_url"] == ""
|
|
assert rows[0]["open_position_url"] == ""
|
|
|
|
def test_complete_rows_sorted_first(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
# Insert incomplete row first, complete row second
|
|
db.persist_result(_result(
|
|
job_id="1", company_name="ZZZ Corp",
|
|
position_url=None, status=JobStatus.needs_review,
|
|
))
|
|
db.persist_result(_result(
|
|
job_id="2", company_name="AAA Inc",
|
|
position_url="https://aaa.com/jobs/1", status=JobStatus.position_found,
|
|
))
|
|
out = tmp_path / "out.csv"
|
|
db.export_csv(path=out)
|
|
with out.open() as f:
|
|
rows = list(csv.DictReader(f))
|
|
assert len(rows) == 2
|
|
# complete row (AAA Inc) must come first
|
|
assert rows[0]["open_position_url"] != ""
|
|
assert rows[1]["open_position_url"] == ""
|
|
|
|
def test_creates_parent_dirs(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.persist_result(_result())
|
|
nested = tmp_path / "deep" / "dir" / "results.csv"
|
|
db.export_csv(path=nested)
|
|
assert nested.exists()
|
|
|
|
def test_returns_path(self, tmp_path: Path) -> None:
|
|
with _db(tmp_path) as db:
|
|
db.persist_result(_result())
|
|
out = tmp_path / "r.csv"
|
|
returned = db.export_csv(path=out)
|
|
assert returned == out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# pipeline.run_batch — offline unit tests via monkeypatch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_raw(job_id: str, company: str = "Acme") -> RawJob:
|
|
return RawJob(
|
|
job_id=job_id,
|
|
company=company,
|
|
linkedin_url=f"https://www.linkedin.com/jobs/view/{job_id}",
|
|
website=None,
|
|
)
|
|
|
|
|
|
class TestRunBatch:
|
|
"""Monkeypatched run_batch tests — no network, no real providers."""
|
|
|
|
def _patch_all(self, monkeypatch, raw_jobs: list[RawJob]) -> None:
|
|
"""Patch all external I/O in pipeline so tests run offline."""
|
|
from jobsource.careers.cascade import CareersResult
|
|
|
|
# Fake job source
|
|
fake_source = MagicMock()
|
|
fake_source.fetch_recent_jobs.return_value = raw_jobs
|
|
monkeypatch.setattr("jobsource.pipeline._make_source", lambda _: fake_source)
|
|
|
|
# Fake resolve: always succeeds
|
|
monkeypatch.setattr(
|
|
"jobsource.pipeline.resolve_website",
|
|
lambda name, website=None, *, client=None: f"https://{name.lower().replace(' ', '')}.com",
|
|
)
|
|
|
|
# Fake cascade: returns a careers URL + free position shortcut
|
|
def _fake_careers(website, *, company_name=None, client=None):
|
|
return CareersResult(
|
|
careers_url=f"{website}/careers",
|
|
confidence=0.95,
|
|
method="ats:greenhouse",
|
|
position_url=f"{website}/careers/jobs/1",
|
|
)
|
|
|
|
monkeypatch.setattr("jobsource.pipeline.find_careers_page", _fake_careers)
|
|
|
|
# extract_open_position shouldn't be called (shortcut covers it),
|
|
# but patch defensively to detect if it is.
|
|
monkeypatch.setattr(
|
|
"jobsource.pipeline.extract_open_position",
|
|
lambda url, *, client=None: (None, "none"),
|
|
)
|
|
|
|
# build_client — return a context-manager stub
|
|
cm = MagicMock()
|
|
cm.__enter__ = lambda s: MagicMock()
|
|
cm.__exit__ = MagicMock(return_value=False)
|
|
monkeypatch.setattr("jobsource.pipeline.build_client", lambda: cm)
|
|
|
|
def test_new_jobs_processed(self, tmp_path: Path, monkeypatch) -> None:
|
|
from jobsource.pipeline import run_batch
|
|
|
|
# Use distinct company names so each job gets its own company_key (no cache sharing).
|
|
raw = [_make_raw("1", company="Alpha"), _make_raw("2", company="Beta")]
|
|
self._patch_all(monkeypatch, raw)
|
|
|
|
db = Database(path=tmp_path / "t.db")
|
|
summary = run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
|
|
assert summary.fetched == 2
|
|
assert summary.new == 2
|
|
assert summary.already_seen == 0
|
|
assert summary.website_resolved == 2
|
|
assert summary.careers_found == 2
|
|
assert summary.position_found == 2
|
|
assert summary.failed == 0
|
|
assert summary.coverage == 1.0
|
|
|
|
def test_cross_run_dedup(self, tmp_path: Path, monkeypatch) -> None:
|
|
"""Re-running with the same DB yields new == 0 (dedup proven)."""
|
|
from jobsource.pipeline import run_batch
|
|
|
|
raw = [_make_raw("10", company="Gamma"), _make_raw("11", company="Delta")]
|
|
self._patch_all(monkeypatch, raw)
|
|
|
|
db = Database(path=tmp_path / "dedup.db")
|
|
|
|
# First run
|
|
s1 = run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
assert s1.new == 2
|
|
|
|
# Second run — same raw jobs returned by source, but DB already has them
|
|
s2 = run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
assert s2.new == 0
|
|
assert s2.already_seen == 2
|
|
|
|
def test_batch_size_cap(self, tmp_path: Path, monkeypatch) -> None:
|
|
"""Only batch_size new jobs are processed even if more are available."""
|
|
from jobsource.pipeline import run_batch
|
|
|
|
# Distinct companies to avoid career-URL cache sharing across jobs.
|
|
raw = [_make_raw(str(i), company=f"Co{i}") for i in range(10)]
|
|
self._patch_all(monkeypatch, raw)
|
|
|
|
db = Database(path=tmp_path / "cap.db")
|
|
summary = run_batch(batch_size=3, search_terms=["eng"], db=db)
|
|
assert summary.new == 3
|
|
assert summary.position_found == 3
|
|
|
|
def test_resolve_failure_marks_needs_review(self, tmp_path: Path, monkeypatch) -> None:
|
|
"""Jobs where resolve_website returns None get needs_review, not failed."""
|
|
from jobsource.pipeline import run_batch
|
|
|
|
raw = [_make_raw("99")]
|
|
self._patch_all(monkeypatch, raw)
|
|
# Override resolve to return None (unresolvable)
|
|
monkeypatch.setattr(
|
|
"jobsource.pipeline.resolve_website",
|
|
lambda *a, **kw: None,
|
|
)
|
|
|
|
db = Database(path=tmp_path / "nr.db")
|
|
summary = run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
assert summary.needs_review == 1
|
|
assert summary.failed == 0
|
|
assert summary.website_resolved == 0
|
|
|
|
def test_csv_written_after_batch(self, tmp_path: Path, monkeypatch) -> None:
|
|
"""Data is persisted by run_batch and export_csv produces the right columns."""
|
|
from jobsource.pipeline import run_batch
|
|
|
|
raw = [_make_raw("7", company="SevenCo")]
|
|
self._patch_all(monkeypatch, raw)
|
|
|
|
csv_path = tmp_path / "out.csv"
|
|
# Pass db explicitly so own_db=False → database is NOT closed by run_batch.
|
|
db = Database(path=tmp_path / "t.db")
|
|
|
|
run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
|
|
# Export to a controlled path and verify the header/contract.
|
|
db.export_csv(path=csv_path)
|
|
|
|
assert csv_path.exists()
|
|
with csv_path.open() as f:
|
|
reader = csv.DictReader(f)
|
|
assert tuple(reader.fieldnames or []) == CSV_COLUMNS
|
|
|
|
def test_one_failure_does_not_abort_batch(self, tmp_path: Path, monkeypatch) -> None:
|
|
"""A cascade exception for one company doesn't abort others.
|
|
|
|
Cascade errors land in needs_review (not failed — they are recoverable/transient).
|
|
Each job has a distinct company so cache cannot mask the flaky call.
|
|
"""
|
|
from jobsource.pipeline import run_batch
|
|
from jobsource.careers.cascade import CareersResult
|
|
|
|
# Distinct companies → distinct company_keys → no cache sharing.
|
|
raw = [
|
|
_make_raw("A", company="AlphaCo"),
|
|
_make_raw("B", company="BetaCo"),
|
|
_make_raw("C", company="GammaCo"),
|
|
]
|
|
self._patch_all(monkeypatch, raw)
|
|
|
|
call_count = {"n": 0}
|
|
|
|
def _flaky_careers(website, *, company_name=None, client=None):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 2:
|
|
raise RuntimeError("simulated cascade crash")
|
|
return CareersResult(
|
|
careers_url=f"{website}/careers",
|
|
confidence=0.9,
|
|
method="url_pattern",
|
|
position_url=f"{website}/careers/jobs/1",
|
|
)
|
|
|
|
monkeypatch.setattr("jobsource.pipeline.find_careers_page", _flaky_careers)
|
|
|
|
db = Database(path=tmp_path / "fault.db")
|
|
summary = run_batch(batch_size=5, search_terms=["eng"], db=db)
|
|
|
|
# All 3 attempted; cascade exception → needs_review (not failed); 2 fully resolved.
|
|
assert summary.new == 3
|
|
assert summary.needs_review == 1
|
|
assert summary.failed == 0
|
|
assert summary.position_found == 2
|