310 lines
13 KiB
Python
310 lines
13 KiB
Python
"""Tests for extract.py (Stage 3) — all network-free via monkeypatching."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from jobsource.careers.ats import ATSBoard, ATSFetch
|
|
from jobsource.careers.classify_llm import Anchor, JobLinkResult
|
|
from jobsource.extract import _find_job_posting_url, _jsonld_job_url, extract_open_position
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fake helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, status_code: int, text: str = "", url: str = "https://example.com") -> None:
|
|
self.status_code = status_code
|
|
self.text = text
|
|
self.url = url
|
|
|
|
|
|
class FakeClient:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _find_job_posting_url (pure function)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestFindJobPostingUrl:
|
|
def test_flat_dict_job_posting(self):
|
|
data = {"@type": "JobPosting", "url": "https://acme.com/jobs/1"}
|
|
assert _find_job_posting_url(data) == "https://acme.com/jobs/1"
|
|
|
|
def test_list_of_nodes(self):
|
|
data = [
|
|
{"@type": "Organization", "name": "Acme"},
|
|
{"@type": "JobPosting", "url": "https://acme.com/jobs/2"},
|
|
]
|
|
assert _find_job_posting_url(data) == "https://acme.com/jobs/2"
|
|
|
|
def test_graph_expansion(self):
|
|
data = {
|
|
"@context": "https://schema.org",
|
|
"@graph": [
|
|
{"@type": "WebPage"},
|
|
{"@type": "JobPosting", "url": "https://acme.com/jobs/3"},
|
|
],
|
|
}
|
|
assert _find_job_posting_url(data) == "https://acme.com/jobs/3"
|
|
|
|
def test_type_as_list(self):
|
|
data = {"@type": ["Thing", "JobPosting"], "url": "https://acme.com/jobs/4"}
|
|
assert _find_job_posting_url(data) == "https://acme.com/jobs/4"
|
|
|
|
def test_no_job_posting_returns_none(self):
|
|
data = {"@type": "Organization", "name": "Acme"}
|
|
assert _find_job_posting_url(data) is None
|
|
|
|
def test_job_posting_without_url_returns_none(self):
|
|
data = {"@type": "JobPosting"}
|
|
assert _find_job_posting_url(data) is None
|
|
|
|
def test_scalar_input_returns_none(self):
|
|
assert _find_job_posting_url("not a dict") is None
|
|
assert _find_job_posting_url(42) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _jsonld_job_url (parses HTML)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestJsonldJobUrl:
|
|
def test_finds_url_in_json_ld_block(self):
|
|
html = """
|
|
<html><head>
|
|
<script type="application/ld+json">
|
|
{"@type": "JobPosting", "url": "https://acme.com/jobs/99"}
|
|
</script>
|
|
</head></html>
|
|
"""
|
|
assert _jsonld_job_url(html) == "https://acme.com/jobs/99"
|
|
|
|
def test_invalid_json_block_skipped(self):
|
|
html = """
|
|
<html><head>
|
|
<script type="application/ld+json">NOT JSON{{</script>
|
|
<script type="application/ld+json">{"@type": "JobPosting", "url": "https://ok.com/job/1"}</script>
|
|
</head></html>
|
|
"""
|
|
assert _jsonld_job_url(html) == "https://ok.com/job/1"
|
|
|
|
def test_no_job_posting_returns_none(self):
|
|
html = """
|
|
<html><head>
|
|
<script type="application/ld+json">{"@type": "Organization"}</script>
|
|
</head></html>
|
|
"""
|
|
assert _jsonld_job_url(html) is None
|
|
|
|
def test_no_script_returns_none(self):
|
|
assert _jsonld_job_url("<html><body>No scripts</body></html>") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# extract_open_position — tier integration (network-free)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestExtractOpenPositionTiers:
|
|
def _patch_base(self, monkeypatch, *, html: str | None = None) -> None:
|
|
"""Patch ATS detect functions to miss and HTTP fetch to return html."""
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats.detect_ats_in_url",
|
|
lambda url: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats.detect_ats_in_html",
|
|
lambda html: None,
|
|
)
|
|
if html is not None:
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: FakeResponse(200, html),
|
|
)
|
|
|
|
# -- Tier 1: ATS from URL ------------------------------------------------
|
|
|
|
def test_ats_from_url_returns_first_job(self, monkeypatch):
|
|
board = ATSBoard(
|
|
ats_name="greenhouse",
|
|
slug="acme",
|
|
careers_url="https://boards.greenhouse.io/acme",
|
|
)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats.detect_ats_in_url",
|
|
lambda url: board,
|
|
)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats._FETCH_DISPATCH",
|
|
{"greenhouse": lambda b, c: ATSFetch(first_url="https://boards.greenhouse.io/acme/jobs/1", job_count=5)},
|
|
)
|
|
url, method = extract_open_position(
|
|
"https://boards.greenhouse.io/acme", client=FakeClient()
|
|
)
|
|
assert url == "https://boards.greenhouse.io/acme/jobs/1"
|
|
assert method == "ats:greenhouse"
|
|
|
|
def test_ats_from_url_with_no_jobs_falls_through(self, monkeypatch):
|
|
board = ATSBoard(
|
|
ats_name="lever",
|
|
slug="acme",
|
|
careers_url="https://jobs.lever.co/acme",
|
|
)
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: board)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats._FETCH_DISPATCH",
|
|
{"lever": lambda b, c: ATSFetch(first_url=None, job_count=0)},
|
|
)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats.detect_ats_in_html", lambda html: None
|
|
)
|
|
# Page has no JSON-LD, no job-path anchors, LLM returns None.
|
|
# Use a neutral href that won't trigger the job-path pattern.
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: FakeResponse(200, "<html><body><a href='/team'>Team</a></body></html>"),
|
|
)
|
|
monkeypatch.setattr("jobsource.extract.classify_job_link", lambda anchors: None)
|
|
url, method = extract_open_position("https://acme.example.com/careers", client=FakeClient())
|
|
assert url is None
|
|
assert method == "none"
|
|
|
|
# -- Tier 1b: ATS from page HTML ----------------------------------------
|
|
|
|
def test_ats_from_html_returns_first_job(self, monkeypatch):
|
|
gh_board = ATSBoard(
|
|
ats_name="greenhouse",
|
|
slug="vercel",
|
|
careers_url="https://boards.greenhouse.io/vercel",
|
|
)
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_html", lambda html: gh_board)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats._FETCH_DISPATCH",
|
|
{"greenhouse": lambda b, c: ATSFetch(first_url="https://boards.greenhouse.io/vercel/jobs/42", job_count=73)},
|
|
)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: FakeResponse(200, '<script src="greenhouse"></script>'),
|
|
)
|
|
url, method = extract_open_position("https://vercel.com/careers", client=FakeClient())
|
|
assert url == "https://boards.greenhouse.io/vercel/jobs/42"
|
|
assert method == "ats:greenhouse"
|
|
|
|
# -- Tier 2: JSON-LD ----------------------------------------------------
|
|
|
|
def test_jsonld_returns_job_url(self, monkeypatch):
|
|
html = """
|
|
<html><head>
|
|
<script type="application/ld+json">
|
|
{"@type": "JobPosting", "url": "https://acme.com/jobs/77"}
|
|
</script>
|
|
</head><body></body></html>
|
|
"""
|
|
self._patch_base(monkeypatch, html=html)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url == "https://acme.com/jobs/77"
|
|
assert method == "json_ld"
|
|
|
|
# -- Tier 3: job-like anchor --------------------------------------------
|
|
|
|
def test_job_anchor_matched_by_pattern(self, monkeypatch):
|
|
html = """
|
|
<html><body>
|
|
<a href="/about">About</a>
|
|
<a href="/jobs/software-engineer-123">Software Engineer</a>
|
|
</body></html>
|
|
"""
|
|
self._patch_base(monkeypatch, html=html)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url == "https://acme.com/jobs/software-engineer-123"
|
|
assert method == "anchor"
|
|
|
|
def test_opening_anchor_matched(self, monkeypatch):
|
|
html = '<html><body><a href="/openings/42">Open Role</a></body></html>'
|
|
self._patch_base(monkeypatch, html=html)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url == "https://acme.com/openings/42"
|
|
assert method == "anchor"
|
|
|
|
# -- Tier 4: LLM --------------------------------------------------------
|
|
|
|
def test_llm_classify_fires_when_no_job_anchor(self, monkeypatch):
|
|
html = '<html><body><a href="/about">About</a><a href="/team">Team</a></body></html>'
|
|
self._patch_base(monkeypatch, html=html)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.classify_job_link",
|
|
lambda anchors: JobLinkResult(job_url="https://acme.com/about", confidence=0.60),
|
|
)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url == "https://acme.com/about"
|
|
assert method == "llm_classify"
|
|
|
|
def test_llm_returns_none_falls_through_to_none(self, monkeypatch):
|
|
html = '<html><body><a href="/about">About</a></body></html>'
|
|
self._patch_base(monkeypatch, html=html)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.classify_job_link",
|
|
lambda anchors: None,
|
|
)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url is None
|
|
assert method == "none"
|
|
|
|
# -- Page fetch failure -------------------------------------------------
|
|
|
|
def test_fetch_failure_returns_none(self, monkeypatch):
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("timeout")),
|
|
)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url is None
|
|
assert method == "none"
|
|
|
|
def test_non_200_fetch_returns_none(self, monkeypatch):
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: FakeResponse(404, ""),
|
|
)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url is None
|
|
assert method == "none"
|
|
|
|
# -- All tiers miss ------------------------------------------------------
|
|
|
|
def test_all_miss_returns_none_none(self, monkeypatch):
|
|
html = '<html><body><a href="/about">About</a></body></html>'
|
|
self._patch_base(monkeypatch, html=html)
|
|
monkeypatch.setattr("jobsource.extract.classify_job_link", lambda anchors: None)
|
|
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
|
assert url is None
|
|
assert method == "none"
|
|
|
|
# -- ATS tier error falls through ---------------------------------------
|
|
|
|
def test_ats_fetch_error_falls_through(self, monkeypatch):
|
|
board = ATSBoard(ats_name="ashby", slug="acme", careers_url="https://jobs.ashbyhq.com/acme")
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: board)
|
|
monkeypatch.setattr(
|
|
"jobsource.extract._ats._FETCH_DISPATCH",
|
|
{"ashby": lambda b, c: (_ for _ in ()).throw(RuntimeError("api down"))},
|
|
)
|
|
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_html", lambda html: None)
|
|
html = '<html><body><a href="/jobs/42">Engineer</a></body></html>'
|
|
monkeypatch.setattr(
|
|
"jobsource.extract.request_with_retries",
|
|
lambda *a, **kw: FakeResponse(200, html),
|
|
)
|
|
url, method = extract_open_position("https://jobs.ashbyhq.com/acme", client=FakeClient())
|
|
# urljoin("https://jobs.ashbyhq.com/acme", "/jobs/42") → root-relative
|
|
assert url == "https://jobs.ashbyhq.com/jobs/42"
|
|
assert method == "anchor"
|