careers-llm-extract
This commit is contained in:
@@ -550,3 +550,112 @@ class TestSlugGuessTier:
|
||||
result = find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert result.method == "url_pattern"
|
||||
assert result.careers_url == "https://acme.com/careers"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tier 5 — Cheap-LLM careers-link classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLLMClassifyTier:
|
||||
"""Tier 5 fires after sitemap, no-ops when LLM returns None, and is skipped on earlier hits."""
|
||||
|
||||
def _patch_all_deterministic_miss(self, monkeypatch, *, homepage_html: str = "<html/>") -> None:
|
||||
"""Patch tiers 1–4 to all miss; leaves Tier 5 for the caller to control."""
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._safe_get_html",
|
||||
lambda website, client: homepage_html,
|
||||
)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.detect_and_fetch", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.recover_via_slug_guess", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._heuristics.probe_url_patterns", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._heuristics.scan_homepage_links", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._heuristics.parse_sitemap", lambda *a, **kw: None)
|
||||
# Prevent _finalize from making network calls
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.detect_ats_in_url", lambda url: None)
|
||||
|
||||
def test_llm_classify_hit_returns_055_confidence(self, monkeypatch):
|
||||
from jobsource.careers.classify_llm import Anchor, CareerLinkResult
|
||||
|
||||
self._patch_all_deterministic_miss(
|
||||
monkeypatch,
|
||||
homepage_html='<html><body><a href="/careers">Careers</a></body></html>',
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.extract_anchors",
|
||||
lambda html, base_url: [Anchor(url="https://acme.com/careers", text="Careers")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.classify_careers_link",
|
||||
lambda anchors: CareerLinkResult(careers_url="https://acme.com/careers", confidence=0.85),
|
||||
)
|
||||
result = find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert result.method == "llm_classify"
|
||||
assert result.confidence == 0.55 # fixed tier-level confidence from cascade
|
||||
assert result.careers_url == "https://acme.com/careers"
|
||||
|
||||
def test_llm_classify_none_falls_through_to_method_none(self, monkeypatch):
|
||||
from jobsource.careers.classify_llm import Anchor
|
||||
|
||||
self._patch_all_deterministic_miss(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.extract_anchors",
|
||||
lambda html, base_url: [Anchor(url="https://acme.com/about", text="About")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.classify_careers_link",
|
||||
lambda anchors: None,
|
||||
)
|
||||
result = find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert result.method == "none"
|
||||
assert result.careers_url is None
|
||||
|
||||
def test_tier5_skipped_when_homepage_html_is_none(self, monkeypatch):
|
||||
"""When the homepage fetch returns None, Tier 5 is silently bypassed."""
|
||||
llm_called: list[bool] = []
|
||||
self._patch_all_deterministic_miss(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._safe_get_html",
|
||||
lambda website, client: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.classify_careers_link",
|
||||
lambda anchors: llm_called.append(True) or None,
|
||||
)
|
||||
find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert llm_called == []
|
||||
|
||||
def test_tier5_not_called_when_sitemap_hits(self, monkeypatch):
|
||||
"""When sitemap resolves, the cascade returns before reaching Tier 5."""
|
||||
llm_called: list[bool] = []
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._safe_get_html",
|
||||
lambda website, client: "<html/>",
|
||||
)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.detect_and_fetch", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.recover_via_slug_guess", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._heuristics.probe_url_patterns", lambda *a, **kw: None)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._heuristics.scan_homepage_links", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._heuristics.parse_sitemap",
|
||||
lambda *a, **kw: "https://acme.com/careers",
|
||||
)
|
||||
monkeypatch.setattr("jobsource.careers.cascade._ats.detect_ats_in_url", lambda url: None)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.classify_careers_link",
|
||||
lambda anchors: llm_called.append(True) or None,
|
||||
)
|
||||
result = find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert result.method == "sitemap"
|
||||
assert llm_called == []
|
||||
|
||||
def test_tier5_error_falls_through_gracefully(self, monkeypatch):
|
||||
"""A Tier 5 exception should not abort the cascade."""
|
||||
self._patch_all_deterministic_miss(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.cascade._classify_llm.extract_anchors",
|
||||
lambda html, base_url: (_ for _ in ()).throw(RuntimeError("classify boom")),
|
||||
)
|
||||
result = find_careers_page("https://www.acme.com", client=FakeClient())
|
||||
assert result.method == "none"
|
||||
assert result.careers_url is None
|
||||
|
||||
256
tests/test_classify_llm.py
Normal file
256
tests/test_classify_llm.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""Tests for careers/classify_llm.py — all network-free via monkeypatching."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from jobsource.careers.classify_llm import (
|
||||
Anchor,
|
||||
CareerLinkResult,
|
||||
JobLinkResult,
|
||||
_LinkChoice,
|
||||
classify_careers_link,
|
||||
classify_job_link,
|
||||
extract_anchors,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
"""Minimal stand-in for a Pydantic AI run result."""
|
||||
|
||||
def __init__(self, output: _LinkChoice) -> None:
|
||||
self.output = output
|
||||
|
||||
|
||||
class _FakeAgent:
|
||||
"""Stand-in for a Pydantic AI Agent; returns a fixed _LinkChoice."""
|
||||
|
||||
def __init__(self, choice: _LinkChoice) -> None:
|
||||
self._choice = choice
|
||||
|
||||
def run_sync(self, prompt: str) -> _FakeResult:
|
||||
return _FakeResult(self._choice)
|
||||
|
||||
|
||||
class _RaisingAgent:
|
||||
"""Stand-in that raises on run_sync, simulating an LLM failure."""
|
||||
|
||||
def run_sync(self, prompt: str) -> _FakeResult:
|
||||
raise RuntimeError("LLM unavailable")
|
||||
|
||||
|
||||
def _real_settings():
|
||||
"""Fake settings object with non-placeholder values."""
|
||||
return type("S", (), {
|
||||
"classifier_model": "openai:gpt-4o-mini",
|
||||
"llm_api_key": "sk-real-key",
|
||||
})()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_anchors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SAMPLE_HTML = """
|
||||
<html><body>
|
||||
<a href="/about">About Us</a>
|
||||
<a href="/careers">Careers</a>
|
||||
<a href="https://external.com/jobs">External Jobs</a>
|
||||
<a href="mailto:info@acme.com">Email</a>
|
||||
<a href="#section">Anchor</a>
|
||||
<a href="javascript:void(0)">JS link</a>
|
||||
<a href="/careers">Duplicate careers link</a>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
class TestExtractAnchors:
|
||||
def test_resolves_relative_to_absolute(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
urls = [a.url for a in anchors]
|
||||
assert "https://acme.com/about" in urls
|
||||
assert "https://acme.com/careers" in urls
|
||||
|
||||
def test_preserves_already_absolute_urls(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
urls = [a.url for a in anchors]
|
||||
assert "https://external.com/jobs" in urls
|
||||
|
||||
def test_skips_mailto(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
assert all("mailto:" not in a.url for a in anchors)
|
||||
|
||||
def test_skips_hash_fragments(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
# A pure # anchor resolves to base_url itself (https://acme.com) —
|
||||
# it starts with http so it passes the scheme check but we skip hrefs
|
||||
# that start with "#". Verify the JavaScript link is also gone.
|
||||
assert all("javascript:" not in a.url for a in anchors)
|
||||
|
||||
def test_deduplicates_by_url(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
urls = [a.url for a in anchors]
|
||||
assert urls.count("https://acme.com/careers") == 1
|
||||
|
||||
def test_captures_anchor_text(self):
|
||||
anchors = extract_anchors(_SAMPLE_HTML, "https://acme.com")
|
||||
career_anchor = next(a for a in anchors if a.url == "https://acme.com/careers")
|
||||
assert career_anchor.text == "Careers"
|
||||
|
||||
def test_empty_html_returns_empty_list(self):
|
||||
assert extract_anchors("", "https://acme.com") == []
|
||||
|
||||
def test_html_with_no_links_returns_empty_list(self):
|
||||
assert extract_anchors("<html><body><p>No links here.</p></body></html>", "https://acme.com") == []
|
||||
|
||||
def test_text_truncated_at_max_length(self):
|
||||
long_text = "x" * 200
|
||||
html = f'<html><body><a href="/careers">{long_text}</a></body></html>'
|
||||
anchors = extract_anchors(html, "https://acme.com")
|
||||
assert len(anchors) == 1
|
||||
assert len(anchors[0].text) <= 80
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graceful degradation: placeholder settings → always None
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPlaceholderGate:
|
||||
"""When classifier_model or llm_api_key is a placeholder, classifiers no-op."""
|
||||
|
||||
def test_classify_careers_link_returns_none_with_placeholder(self):
|
||||
anchors = [Anchor(url="https://acme.com/careers", text="Careers")]
|
||||
assert classify_careers_link(anchors) is None
|
||||
|
||||
def test_classify_job_link_returns_none_with_placeholder(self):
|
||||
anchors = [Anchor(url="https://acme.com/jobs/123", text="Engineer")]
|
||||
assert classify_job_link(anchors) is None
|
||||
|
||||
def test_classify_careers_link_returns_none_on_empty_list(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm.get_settings",
|
||||
_real_settings,
|
||||
)
|
||||
assert classify_careers_link([]) is None
|
||||
|
||||
def test_classify_job_link_returns_none_on_empty_list(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm.get_settings",
|
||||
_real_settings,
|
||||
)
|
||||
assert classify_job_link([]) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Success paths via _build_link_agent monkeypatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClassifySuccess:
|
||||
def _patch_agent(self, monkeypatch, choice: _LinkChoice) -> None:
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm.get_settings",
|
||||
_real_settings,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
lambda instructions: _FakeAgent(choice),
|
||||
)
|
||||
|
||||
def test_careers_pick_returns_mapped_url(self, monkeypatch):
|
||||
anchors = [
|
||||
Anchor(url="https://acme.com/about", text="About"),
|
||||
Anchor(url="https://acme.com/careers", text="Careers"),
|
||||
]
|
||||
self._patch_agent(monkeypatch, _LinkChoice(index=1, confidence=0.95))
|
||||
result = classify_careers_link(anchors)
|
||||
assert result is not None
|
||||
assert isinstance(result, CareerLinkResult)
|
||||
assert result.careers_url == "https://acme.com/careers"
|
||||
assert result.confidence == 0.95
|
||||
|
||||
def test_job_pick_returns_mapped_url(self, monkeypatch):
|
||||
anchors = [
|
||||
Anchor(url="https://acme.com/jobs/", text="All Jobs"),
|
||||
Anchor(url="https://acme.com/jobs/123", text="Software Engineer"),
|
||||
]
|
||||
self._patch_agent(monkeypatch, _LinkChoice(index=1, confidence=0.85))
|
||||
result = classify_job_link(anchors)
|
||||
assert result is not None
|
||||
assert isinstance(result, JobLinkResult)
|
||||
assert result.job_url == "https://acme.com/jobs/123"
|
||||
assert result.confidence == 0.85
|
||||
|
||||
def test_careers_first_index_zero(self, monkeypatch):
|
||||
anchors = [Anchor(url="https://acme.com/jobs", text="Jobs")]
|
||||
self._patch_agent(monkeypatch, _LinkChoice(index=0, confidence=0.70))
|
||||
result = classify_careers_link(anchors)
|
||||
assert result is not None
|
||||
assert result.careers_url == "https://acme.com/jobs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Abstain and failure paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClassifyFailurePaths:
|
||||
def _patch_real_settings(self, monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm.get_settings",
|
||||
_real_settings,
|
||||
)
|
||||
|
||||
def test_abstain_index_minus_one_returns_none(self, monkeypatch):
|
||||
self._patch_real_settings(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
lambda instructions: _FakeAgent(_LinkChoice(index=-1, confidence=0.0)),
|
||||
)
|
||||
anchors = [Anchor(url="https://acme.com/about", text="About")]
|
||||
assert classify_careers_link(anchors) is None
|
||||
|
||||
def test_out_of_range_index_returns_none(self, monkeypatch):
|
||||
self._patch_real_settings(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
lambda instructions: _FakeAgent(_LinkChoice(index=99, confidence=0.9)),
|
||||
)
|
||||
anchors = [Anchor(url="https://acme.com/careers", text="Careers")]
|
||||
assert classify_careers_link(anchors) is None
|
||||
|
||||
def test_run_sync_exception_returns_none(self, monkeypatch):
|
||||
self._patch_real_settings(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
lambda instructions: _RaisingAgent(),
|
||||
)
|
||||
anchors = [Anchor(url="https://acme.com/careers", text="Careers")]
|
||||
assert classify_careers_link(anchors) is None
|
||||
|
||||
def test_build_agent_exception_returns_none(self, monkeypatch):
|
||||
self._patch_real_settings(monkeypatch)
|
||||
|
||||
def boom(instructions: str):
|
||||
raise RuntimeError("import error")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
boom,
|
||||
)
|
||||
anchors = [Anchor(url="https://acme.com/careers", text="Careers")]
|
||||
assert classify_careers_link(anchors) is None
|
||||
|
||||
def test_job_classify_run_sync_exception_returns_none(self, monkeypatch):
|
||||
self._patch_real_settings(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.careers.classify_llm._build_link_agent",
|
||||
lambda instructions: _RaisingAgent(),
|
||||
)
|
||||
anchors = [Anchor(url="https://acme.com/jobs/123", text="Engineer")]
|
||||
assert classify_job_link(anchors) is None
|
||||
309
tests/test_extract.py
Normal file
309
tests/test_extract.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""Tests for extract.py (Stage 3) — all network-free via monkeypatching."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from jobsource.careers.ats import ATSBoard, ATSFetch
|
||||
from jobsource.careers.classify_llm import Anchor, JobLinkResult
|
||||
from jobsource.extract import _find_job_posting_url, _jsonld_job_url, extract_open_position
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, status_code: int, text: str = "", url: str = "https://example.com") -> None:
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
self.url = url
|
||||
|
||||
|
||||
class FakeClient:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _find_job_posting_url (pure function)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFindJobPostingUrl:
|
||||
def test_flat_dict_job_posting(self):
|
||||
data = {"@type": "JobPosting", "url": "https://acme.com/jobs/1"}
|
||||
assert _find_job_posting_url(data) == "https://acme.com/jobs/1"
|
||||
|
||||
def test_list_of_nodes(self):
|
||||
data = [
|
||||
{"@type": "Organization", "name": "Acme"},
|
||||
{"@type": "JobPosting", "url": "https://acme.com/jobs/2"},
|
||||
]
|
||||
assert _find_job_posting_url(data) == "https://acme.com/jobs/2"
|
||||
|
||||
def test_graph_expansion(self):
|
||||
data = {
|
||||
"@context": "https://schema.org",
|
||||
"@graph": [
|
||||
{"@type": "WebPage"},
|
||||
{"@type": "JobPosting", "url": "https://acme.com/jobs/3"},
|
||||
],
|
||||
}
|
||||
assert _find_job_posting_url(data) == "https://acme.com/jobs/3"
|
||||
|
||||
def test_type_as_list(self):
|
||||
data = {"@type": ["Thing", "JobPosting"], "url": "https://acme.com/jobs/4"}
|
||||
assert _find_job_posting_url(data) == "https://acme.com/jobs/4"
|
||||
|
||||
def test_no_job_posting_returns_none(self):
|
||||
data = {"@type": "Organization", "name": "Acme"}
|
||||
assert _find_job_posting_url(data) is None
|
||||
|
||||
def test_job_posting_without_url_returns_none(self):
|
||||
data = {"@type": "JobPosting"}
|
||||
assert _find_job_posting_url(data) is None
|
||||
|
||||
def test_scalar_input_returns_none(self):
|
||||
assert _find_job_posting_url("not a dict") is None
|
||||
assert _find_job_posting_url(42) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _jsonld_job_url (parses HTML)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJsonldJobUrl:
|
||||
def test_finds_url_in_json_ld_block(self):
|
||||
html = """
|
||||
<html><head>
|
||||
<script type="application/ld+json">
|
||||
{"@type": "JobPosting", "url": "https://acme.com/jobs/99"}
|
||||
</script>
|
||||
</head></html>
|
||||
"""
|
||||
assert _jsonld_job_url(html) == "https://acme.com/jobs/99"
|
||||
|
||||
def test_invalid_json_block_skipped(self):
|
||||
html = """
|
||||
<html><head>
|
||||
<script type="application/ld+json">NOT JSON{{</script>
|
||||
<script type="application/ld+json">{"@type": "JobPosting", "url": "https://ok.com/job/1"}</script>
|
||||
</head></html>
|
||||
"""
|
||||
assert _jsonld_job_url(html) == "https://ok.com/job/1"
|
||||
|
||||
def test_no_job_posting_returns_none(self):
|
||||
html = """
|
||||
<html><head>
|
||||
<script type="application/ld+json">{"@type": "Organization"}</script>
|
||||
</head></html>
|
||||
"""
|
||||
assert _jsonld_job_url(html) is None
|
||||
|
||||
def test_no_script_returns_none(self):
|
||||
assert _jsonld_job_url("<html><body>No scripts</body></html>") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_open_position — tier integration (network-free)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractOpenPositionTiers:
|
||||
def _patch_base(self, monkeypatch, *, html: str | None = None) -> None:
|
||||
"""Patch ATS detect functions to miss and HTTP fetch to return html."""
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats.detect_ats_in_url",
|
||||
lambda url: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats.detect_ats_in_html",
|
||||
lambda html: None,
|
||||
)
|
||||
if html is not None:
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: FakeResponse(200, html),
|
||||
)
|
||||
|
||||
# -- Tier 1: ATS from URL ------------------------------------------------
|
||||
|
||||
def test_ats_from_url_returns_first_job(self, monkeypatch):
|
||||
board = ATSBoard(
|
||||
ats_name="greenhouse",
|
||||
slug="acme",
|
||||
careers_url="https://boards.greenhouse.io/acme",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats.detect_ats_in_url",
|
||||
lambda url: board,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats._FETCH_DISPATCH",
|
||||
{"greenhouse": lambda b, c: ATSFetch(first_url="https://boards.greenhouse.io/acme/jobs/1", job_count=5)},
|
||||
)
|
||||
url, method = extract_open_position(
|
||||
"https://boards.greenhouse.io/acme", client=FakeClient()
|
||||
)
|
||||
assert url == "https://boards.greenhouse.io/acme/jobs/1"
|
||||
assert method == "ats:greenhouse"
|
||||
|
||||
def test_ats_from_url_with_no_jobs_falls_through(self, monkeypatch):
|
||||
board = ATSBoard(
|
||||
ats_name="lever",
|
||||
slug="acme",
|
||||
careers_url="https://jobs.lever.co/acme",
|
||||
)
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: board)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats._FETCH_DISPATCH",
|
||||
{"lever": lambda b, c: ATSFetch(first_url=None, job_count=0)},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats.detect_ats_in_html", lambda html: None
|
||||
)
|
||||
# Page has no JSON-LD, no job-path anchors, LLM returns None.
|
||||
# Use a neutral href that won't trigger the job-path pattern.
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: FakeResponse(200, "<html><body><a href='/team'>Team</a></body></html>"),
|
||||
)
|
||||
monkeypatch.setattr("jobsource.extract.classify_job_link", lambda anchors: None)
|
||||
url, method = extract_open_position("https://acme.example.com/careers", client=FakeClient())
|
||||
assert url is None
|
||||
assert method == "none"
|
||||
|
||||
# -- Tier 1b: ATS from page HTML ----------------------------------------
|
||||
|
||||
def test_ats_from_html_returns_first_job(self, monkeypatch):
|
||||
gh_board = ATSBoard(
|
||||
ats_name="greenhouse",
|
||||
slug="vercel",
|
||||
careers_url="https://boards.greenhouse.io/vercel",
|
||||
)
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_html", lambda html: gh_board)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats._FETCH_DISPATCH",
|
||||
{"greenhouse": lambda b, c: ATSFetch(first_url="https://boards.greenhouse.io/vercel/jobs/42", job_count=73)},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: FakeResponse(200, '<script src="greenhouse"></script>'),
|
||||
)
|
||||
url, method = extract_open_position("https://vercel.com/careers", client=FakeClient())
|
||||
assert url == "https://boards.greenhouse.io/vercel/jobs/42"
|
||||
assert method == "ats:greenhouse"
|
||||
|
||||
# -- Tier 2: JSON-LD ----------------------------------------------------
|
||||
|
||||
def test_jsonld_returns_job_url(self, monkeypatch):
|
||||
html = """
|
||||
<html><head>
|
||||
<script type="application/ld+json">
|
||||
{"@type": "JobPosting", "url": "https://acme.com/jobs/77"}
|
||||
</script>
|
||||
</head><body></body></html>
|
||||
"""
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url == "https://acme.com/jobs/77"
|
||||
assert method == "json_ld"
|
||||
|
||||
# -- Tier 3: job-like anchor --------------------------------------------
|
||||
|
||||
def test_job_anchor_matched_by_pattern(self, monkeypatch):
|
||||
html = """
|
||||
<html><body>
|
||||
<a href="/about">About</a>
|
||||
<a href="/jobs/software-engineer-123">Software Engineer</a>
|
||||
</body></html>
|
||||
"""
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url == "https://acme.com/jobs/software-engineer-123"
|
||||
assert method == "anchor"
|
||||
|
||||
def test_opening_anchor_matched(self, monkeypatch):
|
||||
html = '<html><body><a href="/openings/42">Open Role</a></body></html>'
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url == "https://acme.com/openings/42"
|
||||
assert method == "anchor"
|
||||
|
||||
# -- Tier 4: LLM --------------------------------------------------------
|
||||
|
||||
def test_llm_classify_fires_when_no_job_anchor(self, monkeypatch):
|
||||
html = '<html><body><a href="/about">About</a><a href="/team">Team</a></body></html>'
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.classify_job_link",
|
||||
lambda anchors: JobLinkResult(job_url="https://acme.com/about", confidence=0.60),
|
||||
)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url == "https://acme.com/about"
|
||||
assert method == "llm_classify"
|
||||
|
||||
def test_llm_returns_none_falls_through_to_none(self, monkeypatch):
|
||||
html = '<html><body><a href="/about">About</a></body></html>'
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.classify_job_link",
|
||||
lambda anchors: None,
|
||||
)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url is None
|
||||
assert method == "none"
|
||||
|
||||
# -- Page fetch failure -------------------------------------------------
|
||||
|
||||
def test_fetch_failure_returns_none(self, monkeypatch):
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("timeout")),
|
||||
)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url is None
|
||||
assert method == "none"
|
||||
|
||||
def test_non_200_fetch_returns_none(self, monkeypatch):
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: None)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: FakeResponse(404, ""),
|
||||
)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url is None
|
||||
assert method == "none"
|
||||
|
||||
# -- All tiers miss ------------------------------------------------------
|
||||
|
||||
def test_all_miss_returns_none_none(self, monkeypatch):
|
||||
html = '<html><body><a href="/about">About</a></body></html>'
|
||||
self._patch_base(monkeypatch, html=html)
|
||||
monkeypatch.setattr("jobsource.extract.classify_job_link", lambda anchors: None)
|
||||
url, method = extract_open_position("https://acme.com/careers", client=FakeClient())
|
||||
assert url is None
|
||||
assert method == "none"
|
||||
|
||||
# -- ATS tier error falls through ---------------------------------------
|
||||
|
||||
def test_ats_fetch_error_falls_through(self, monkeypatch):
|
||||
board = ATSBoard(ats_name="ashby", slug="acme", careers_url="https://jobs.ashbyhq.com/acme")
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_url", lambda url: board)
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract._ats._FETCH_DISPATCH",
|
||||
{"ashby": lambda b, c: (_ for _ in ()).throw(RuntimeError("api down"))},
|
||||
)
|
||||
monkeypatch.setattr("jobsource.extract._ats.detect_ats_in_html", lambda html: None)
|
||||
html = '<html><body><a href="/jobs/42">Engineer</a></body></html>'
|
||||
monkeypatch.setattr(
|
||||
"jobsource.extract.request_with_retries",
|
||||
lambda *a, **kw: FakeResponse(200, html),
|
||||
)
|
||||
url, method = extract_open_position("https://jobs.ashbyhq.com/acme", client=FakeClient())
|
||||
# urljoin("https://jobs.ashbyhq.com/acme", "/jobs/42") → root-relative
|
||||
assert url == "https://jobs.ashbyhq.com/jobs/42"
|
||||
assert method == "anchor"
|
||||
Reference in New Issue
Block a user