phase1-ingest-resolve

2026-06-17 13:59:00 -04:00
parent f13b8fc1ca
commit cd9ab9b95e
8 changed files with 958 additions and 35 deletions
--- a/tests/test_resolve.py
+++ b/tests/test_resolve.py
@@ -0,0 +1,145 @@
+"""Tests for jobsource/resolve.py — all network-free via monkeypatched _verify."""
+from __future__ import annotations
+
+import pytest
+
+from jobsource.config import get_settings
+from jobsource.resolve import (
+    _search_api_lookup,
+    _slug,
+    _verify,
+    resolve_website,
+)
+
+
+# ---------------------------------------------------------------------------
+# _slug
+# ---------------------------------------------------------------------------
+
+
+class TestSlug:
+    def test_basic(self):
+        assert _slug("GitHub") == "github"
+
+    def test_strips_legal_suffix_inc(self):
+        assert _slug("Acme Inc") == "acme"
+
+    def test_strips_legal_suffix_llc(self):
+        assert _slug("Widgets LLC") == "widgets"
+
+    def test_strips_legal_suffix_corp(self):
+        assert _slug("MegaCorp Corp") == "megacorp"
+
+    def test_strips_multiple_words(self):
+        assert _slug("Some Company Ltd") == "somecompany"
+
+    def test_removes_spaces_and_punctuation(self):
+        assert _slug("Foo & Bar") == "foobar"
+
+    def test_empty_after_strip_returns_none(self):
+        assert _slug("LLC") is None
+
+    def test_empty_string_returns_none(self):
+        assert _slug("") is None
+
+    def test_gmbh(self):
+        assert _slug("Acme GmbH") == "acme"
+
+
+# ---------------------------------------------------------------------------
+# resolve_website — tier 1: provider-supplied
+# ---------------------------------------------------------------------------
+
+
+class TestResolveWebsiteTier1:
+    def test_returns_provider_website_unchanged_if_has_scheme(self, monkeypatch):
+        called = []
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: called.append(u) or None)
+        result = resolve_website("Acme", "https://acme.com")
+        assert result == "https://acme.com"
+        assert called == []  # no network call
+
+    def test_adds_https_if_no_scheme(self, monkeypatch):
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: None)
+        result = resolve_website("Acme", "acme.com")
+        assert result == "https://acme.com"
+
+    def test_placeholder_website_skips_to_next_tier(self, monkeypatch):
+        verify_calls = []
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: verify_calls.append(u) or None)
+        result = resolve_website("Acme", "PLACEHOLDER_URL")
+        assert result is None
+        assert len(verify_calls) >= 1  # fell through to tier 2
+
+
+# ---------------------------------------------------------------------------
+# resolve_website — tier 2: slug guess
+# ---------------------------------------------------------------------------
+
+
+class TestResolveWebsiteTier2:
+    def test_verified_slug_returned(self, monkeypatch):
+        monkeypatch.setattr(
+            "jobsource.resolve._verify",
+            lambda c, u: "https://github.com/" if "github" in u else None,
+        )
+        result = resolve_website("GitHub")
+        assert result == "https://github.com/"
+
+    def test_miss_returns_none_when_search_disabled(self, monkeypatch):
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: None)
+        result = resolve_website("Acme Corp")
+        assert result is None
+
+    def test_unslugable_name_skips_tier2(self, monkeypatch):
+        verify_calls = []
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: verify_calls.append(u) or None)
+        result = resolve_website("LLC")  # slug → None
+        assert result is None
+        assert verify_calls == []
+
+
+# ---------------------------------------------------------------------------
+# resolve_website — tier 3: search API (gated stub)
+# ---------------------------------------------------------------------------
+
+
+class TestResolveWebsiteTier3:
+    def test_search_api_stub_returns_none(self, monkeypatch):
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: None)
+        get_settings.cache_clear()
+        monkeypatch.setenv("SEARCH_API_ENABLED", "true")
+        monkeypatch.setenv("SEARCH_API_KEY", "real-key-abc")
+        get_settings.cache_clear()
+
+        lookup_called = []
+
+        def fake_lookup(name, client):
+            lookup_called.append(name)
+            return None  # stub
+
+        monkeypatch.setattr("jobsource.resolve._search_api_lookup", fake_lookup)
+        result = resolve_website("Some Obscure Co")
+        assert result is None
+        assert lookup_called == ["Some Obscure Co"]
+        get_settings.cache_clear()
+
+    def test_search_api_disabled_by_default(self, monkeypatch):
+        monkeypatch.setattr("jobsource.resolve._verify", lambda c, u: None)
+        lookup_called = []
+        monkeypatch.setattr(
+            "jobsource.resolve._search_api_lookup",
+            lambda name, client: lookup_called.append(name) or None,
+        )
+        resolve_website("Acme")
+        assert lookup_called == []
+
+
+# ---------------------------------------------------------------------------
+# _search_api_lookup stub contract
+# ---------------------------------------------------------------------------
+
+
+class TestSearchApiLookupStub:
+    def test_returns_none(self):
+        assert _search_api_lookup("Acme", None) is None  # type: ignore[arg-type]
--- a/tests/test_sources.py
+++ b/tests/test_sources.py
@@ -0,0 +1,333 @@
+"""Tests for sources/base.py, jobspy_source.py, apify_source.py, and the factory.
+
+All tests are network-free. Heavy provider deps (jobspy, apify-client) are never
+imported; their integration points (_scrape, _run_actor) are monkeypatched.
+"""
+from __future__ import annotations
+
+from datetime import date, datetime
+
+import pytest
+
+from jobsource.config import get_settings
+from jobsource.sources import JobSource, get_job_source
+from jobsource.sources.apify_source import ApifySource
+from jobsource.sources.base import (
+    canonical_linkedin_url,
+    clean_value,
+    parse_linkedin_job_id,
+)
+from jobsource.sources.jobspy_source import JobSpySource, _to_datetime
+
+
+# ---------------------------------------------------------------------------
+# base helpers
+# ---------------------------------------------------------------------------
+
+
+class TestParseLinkedinJobId:
+    def test_clean_url(self):
+        assert parse_linkedin_job_id("https://www.linkedin.com/jobs/view/1234567890") == "1234567890"
+
+    def test_trailing_slash(self):
+        assert parse_linkedin_job_id("https://www.linkedin.com/jobs/view/999/") == "999"
+
+    def test_tracking_params_ignored(self):
+        url = "https://www.linkedin.com/jobs/view/42?refId=abc&trackingId=xyz"
+        assert parse_linkedin_job_id(url) == "42"
+
+    def test_none_input(self):
+        assert parse_linkedin_job_id(None) is None
+
+    def test_non_job_url(self):
+        assert parse_linkedin_job_id("https://www.linkedin.com/company/acme") is None
+
+    def test_empty_string(self):
+        assert parse_linkedin_job_id("") is None
+
+
+class TestCanonicalLinkedinUrl:
+    def test_formats_correctly(self):
+        assert canonical_linkedin_url("123") == "https://www.linkedin.com/jobs/view/123"
+
+
+class TestCleanValue:
+    def test_none(self):
+        assert clean_value(None) is None
+
+    def test_empty_string(self):
+        assert clean_value("") is None
+
+    def test_whitespace(self):
+        assert clean_value("   ") is None
+
+    def test_nan(self):
+        assert clean_value(float("nan")) is None
+
+    def test_normal_string(self):
+        assert clean_value("  Acme Corp  ") == "Acme Corp"
+
+    def test_non_string_coerced(self):
+        assert clean_value(42) == "42"
+
+    def test_zero_is_kept(self):
+        assert clean_value(0) == "0"
+
+
+# ---------------------------------------------------------------------------
+# _to_datetime (module-level helper in jobspy_source)
+# ---------------------------------------------------------------------------
+
+
+class TestToDatetime:
+    def test_none(self):
+        assert _to_datetime(None) is None
+
+    def test_nan(self):
+        assert _to_datetime(float("nan")) is None
+
+    def test_datetime(self):
+        dt = datetime(2024, 1, 15, 12, 0)
+        assert _to_datetime(dt) == dt
+
+    def test_date(self):
+        result = _to_datetime(date(2024, 1, 15))
+        assert result == datetime(2024, 1, 15)
+
+    def test_iso_string(self):
+        assert _to_datetime("2024-01-15") == datetime(2024, 1, 15)
+
+    def test_bad_string(self):
+        assert _to_datetime("not a date") is None
+
+    def test_nat_type_name(self):
+        class FakeNaT:
+            __name__ = "NaTType"
+
+        obj = FakeNaT()
+        type(obj).__name__ = "NaTType"
+        assert _to_datetime(obj) is None
+
+
+# ---------------------------------------------------------------------------
+# JobSpySource._to_raw_job
+# ---------------------------------------------------------------------------
+
+
+class TestJobSpyToRawJob:
+    def _src(self):
+        return JobSpySource()
+
+    def _record(self, **overrides) -> dict:
+        base = {
+            "job_url": "https://www.linkedin.com/jobs/view/100",
+            "company": "Acme Corp",
+            "company_url_direct": "https://acme.com",
+            "date_posted": "2024-06-01",
+            "title": "Engineer",
+            "location": "Remote",
+            "id": None,
+        }
+        base.update(overrides)
+        return base
+
+    def test_basic_mapping(self):
+        raw = self._src()._to_raw_job(self._record())
+        assert raw is not None
+        assert raw.job_id == "100"
+        assert raw.company == "Acme Corp"
+        assert raw.website == "https://acme.com"
+        assert raw.linkedin_url == "https://www.linkedin.com/jobs/view/100"
+        assert raw.listed_at == datetime(2024, 6, 1)
+        assert raw.title == "Engineer"
+        assert raw.location == "Remote"
+
+    def test_website_from_company_url_direct_not_company_url(self):
+        record = self._record(company_url_direct=None, company_url="https://linkedin.com/company/acme")
+        raw = self._src()._to_raw_job(record)
+        assert raw is not None
+        assert raw.website is None  # company_url (LinkedIn page) must NOT be used
+
+    def test_nan_website_becomes_none(self):
+        raw = self._src()._to_raw_job(self._record(company_url_direct=float("nan")))
+        assert raw is not None
+        assert raw.website is None
+
+    def test_missing_job_id_returns_none(self):
+        record = self._record(job_url="https://example.com/not-a-linkedin-url", id=None)
+        assert self._src()._to_raw_job(record) is None
+
+    def test_bare_id_fallback(self):
+        record = self._record(job_url=None, id="999")
+        raw = self._src()._to_raw_job(record)
+        assert raw is not None
+        assert raw.job_id == "999"
+
+    def test_missing_company_returns_none(self):
+        raw = self._src()._to_raw_job(self._record(company=None))
+        assert raw is None
+
+    def test_linkedin_url_is_canonical(self):
+        record = self._record(job_url="https://www.linkedin.com/jobs/view/55?tracking=abc")
+        raw = self._src()._to_raw_job(record)
+        assert raw is not None
+        assert raw.linkedin_url == "https://www.linkedin.com/jobs/view/55"
+        assert "tracking" not in raw.linkedin_url
+
+
+# ---------------------------------------------------------------------------
+# JobSpySource.fetch_recent_jobs (monkeypatched _scrape)
+# ---------------------------------------------------------------------------
+
+
+class TestJobSpyFetchRecentJobs:
+    def _make_record(self, job_id: str, term_suffix: str = "") -> dict:
+        return {
+            "job_url": f"https://www.linkedin.com/jobs/view/{job_id}",
+            "company": f"Acme{term_suffix}",
+            "company_url_direct": None,
+            "date_posted": None,
+            "title": "Eng",
+            "location": "Remote",
+            "id": None,
+        }
+
+    def test_dedup_across_terms(self, monkeypatch):
+        src = JobSpySource()
+        calls = []
+
+        def fake_scrape(term, location, hours_old, results_wanted):
+            calls.append(term)
+            if term == "engineer":
+                return [self._make_record("1"), self._make_record("2")]
+            # "developer" returns job 2 again + a new job 3
+            return [self._make_record("2"), self._make_record("3")]
+
+        monkeypatch.setattr(src, "_scrape", fake_scrape)
+        results = src.fetch_recent_jobs(["engineer", "developer"], "US", 72, 10)
+        ids = {r.job_id for r in results}
+        assert ids == {"1", "2", "3"}  # deduped; "2" not duplicated
+        assert len(calls) == 2
+
+    def test_failing_scrape_returns_empty(self, monkeypatch):
+        src = JobSpySource()
+        monkeypatch.setattr(src, "_scrape", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")))
+        results = src.fetch_recent_jobs(["engineer"], "US", 72, 10)
+        assert results == []
+
+    def test_empty_scrape(self, monkeypatch):
+        src = JobSpySource()
+        monkeypatch.setattr(src, "_scrape", lambda *a, **k: [])
+        results = src.fetch_recent_jobs(["engineer"], "US", 72, 10)
+        assert results == []
+
+
+# ---------------------------------------------------------------------------
+# ApifySource._to_raw_job
+# ---------------------------------------------------------------------------
+
+
+class TestApifyToRawJob:
+    def _src(self):
+        return ApifySource()
+
+    def test_camel_case_keys(self):
+        item = {
+            "jobUrl": "https://www.linkedin.com/jobs/view/77",
+            "companyName": "BigCo",
+            "companyWebsite": "https://bigco.com",
+            "postedAt": "2024-03-01T00:00:00",
+            "title": "PM",
+            "location": "NYC",
+        }
+        raw = self._src()._to_raw_job(item)
+        assert raw is not None
+        assert raw.job_id == "77"
+        assert raw.company == "BigCo"
+        assert raw.website == "https://bigco.com"
+        assert raw.title == "PM"
+
+    def test_snake_case_keys(self):
+        item = {
+            "job_url": "https://www.linkedin.com/jobs/view/88",
+            "company": "LilCo",
+            "website": "https://lilco.io",
+            "date_posted": "2024-04-01",
+            "title": "SWE",
+            "location": "SF",
+        }
+        raw = self._src()._to_raw_job(item)
+        assert raw is not None
+        assert raw.job_id == "88"
+        assert raw.website == "https://lilco.io"
+
+    def test_no_linkedin_url_returns_none(self):
+        item = {"url": "https://example.com/job/99", "company": "X"}
+        assert self._src()._to_raw_job(item) is None
+
+    def test_no_company_returns_none(self):
+        item = {"jobUrl": "https://www.linkedin.com/jobs/view/55"}
+        assert self._src()._to_raw_job(item) is None
+
+
+# ---------------------------------------------------------------------------
+# ApifySource.fetch_recent_jobs (monkeypatched _run_actor)
+# ---------------------------------------------------------------------------
+
+
+class TestApifyFetchRecentJobs:
+    def test_placeholder_token_returns_empty(self):
+        # Default settings have placeholder token
+        src = ApifySource()
+        results = src.fetch_recent_jobs(["engineer"], "US", 72, 10)
+        assert results == []
+
+    def test_run_actor_failure_returns_empty(self, monkeypatch):
+        src = ApifySource()
+        monkeypatch.setattr(get_settings(), "apify_token", "real-token-abc")
+        monkeypatch.setattr(src, "_run_actor", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("api error")))
+        results = src.fetch_recent_jobs(["engineer"], "US", 72, 10)
+        assert results == []
+        get_settings.cache_clear()
+
+    def test_dedup(self, monkeypatch):
+        src = ApifySource()
+        monkeypatch.setattr(get_settings(), "apify_token", "real-token")
+        items = [
+            {"jobUrl": f"https://www.linkedin.com/jobs/view/{i}", "company": "Co"}
+            for i in [10, 10, 20]
+        ]
+        monkeypatch.setattr(src, "_run_actor", lambda *a, **k: items)
+        results = src.fetch_recent_jobs(["eng"], "US", 72, 10)
+        assert {r.job_id for r in results} == {"10", "20"}
+        get_settings.cache_clear()
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+
+class TestGetJobSource:
+    def test_default_returns_jobspy(self):
+        src = get_job_source()
+        assert isinstance(src, JobSpySource)
+
+    def test_apify_returns_apify(self, monkeypatch):
+        get_settings.cache_clear()
+        monkeypatch.setenv("JOB_SOURCE", "apify")
+        get_settings.cache_clear()
+        src = get_job_source(get_settings())
+        assert isinstance(src, ApifySource)
+        get_settings.cache_clear()
+
+    def test_unknown_raises(self, monkeypatch):
+        get_settings.cache_clear()
+        monkeypatch.setenv("JOB_SOURCE", "indeed")
+        get_settings.cache_clear()
+        with pytest.raises(ValueError, match="indeed"):
+            get_job_source(get_settings())
+        get_settings.cache_clear()
+
+    def test_returns_job_source_abc(self):
+        assert isinstance(get_job_source(), JobSource)