careers-llm-extract
This commit is contained in:
@@ -1,12 +1,214 @@
|
||||
"""Extract one open position URL from a careers page (Stage 3).
|
||||
|
||||
Scaffold stub -- not implemented yet.
|
||||
Cascade order (return early on first hit):
|
||||
1. ATS JSON (URL) — detect ATS board coordinates in the careers URL itself;
|
||||
call the board's public JSON API for the first posting URL.
|
||||
1b. ATS JSON (HTML) — fetch the careers page; detect ATS board in its HTML;
|
||||
call the board API. Handles inline JS embeds.
|
||||
2. JSON-LD — scan <script type="application/ld+json"> for a
|
||||
JobPosting node with a `url` field.
|
||||
3. Job-like anchors — first <a> whose href matches /job|/position|/opening|/vacanc.
|
||||
4. Cheap-LLM — classify_job_link() from classify_llm; no-ops gracefully
|
||||
when the LLM is not configured.
|
||||
5. Miss — return (None, "none").
|
||||
|
||||
Returns (url: str | None, method: str). Never raises — any tier failure logs
|
||||
and falls through. method is one of:
|
||||
"ats:{name}" | "json_ld" | "anchor" | "llm_classify" | "none"
|
||||
"""
|
||||
# TODO (Stage 3): implement per CLAUDE.md "Stage 3 — Extract one open position (return on first hit)".
|
||||
# Cascade order (return early on first hit):
|
||||
# 1. ATS JSON — if ATS is already known from Stage 2, return first posting URL directly.
|
||||
# 2. JobPosting JSON-LD — parse application/ld+json for a `url` field.
|
||||
# 3. Job-like anchors — first <a> matching /job, /position, /opening, /vacancy in href.
|
||||
# 4. Cheap-LLM classification — Pydantic AI typed output (classifier_model).
|
||||
# 5. Browser-agent fallback — handled inside the fused Stage-2 agent call in agent_fallback.py.
|
||||
# Returns (url: str | None, method: str) so callers know which tier resolved it.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .careers import ats as _ats
|
||||
from .careers.classify_llm import classify_job_link, extract_anchors
|
||||
from .http import build_client, request_with_retries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Matches href paths that indicate a single-job detail page.
|
||||
_JOB_HREF_RE = re.compile(r"/(job|position|opening|vacanc)", re.IGNORECASE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_html(url: str, client: httpx.Client) -> str | None:
|
||||
"""Fetch page HTML; return text on success or None on any error/non-2xx."""
|
||||
try:
|
||||
resp = request_with_retries(client, "GET", url, max_retries=1)
|
||||
if resp.status_code < 400:
|
||||
return resp.text
|
||||
logger.info("extract._get_html(%s): HTTP %s", url, resp.status_code)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.info("extract._get_html(%s): fetch error: %s", url, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _jsonld_job_url(html: str) -> str | None:
|
||||
"""Scan every <script type="application/ld+json"> block for a JobPosting url."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for script in soup.find_all("script", type="application/ld+json"):
|
||||
try:
|
||||
data = json.loads(script.get_text(strip=True))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
url = _find_job_posting_url(data)
|
||||
if url:
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def _find_job_posting_url(data: object) -> str | None:
|
||||
"""Walk a JSON-LD structure (dict, list, @graph) for a JobPosting url field."""
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
result = _find_job_posting_url(item)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
# Expand @graph (JSON-LD named graph — a list of typed nodes)
|
||||
graph = data.get("@graph")
|
||||
if graph:
|
||||
result = _find_job_posting_url(graph)
|
||||
if result:
|
||||
return result
|
||||
# Check this node's @type
|
||||
type_val = data.get("@type", "")
|
||||
if isinstance(type_val, list):
|
||||
is_job = "JobPosting" in type_val
|
||||
else:
|
||||
is_job = type_val == "JobPosting"
|
||||
if is_job:
|
||||
url = data.get("url")
|
||||
if url and isinstance(url, str):
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_ats_position(board: "_ats.ATSBoard", client: httpx.Client) -> str | None:
|
||||
"""Call the ATS public JSON API for *board* and return the first job URL or None."""
|
||||
fetch_fn = _ats._FETCH_DISPATCH.get(board.ats_name)
|
||||
if fetch_fn is None:
|
||||
return None
|
||||
fetch = fetch_fn(board, client) # type: ignore[operator]
|
||||
return fetch.first_url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_open_position(
|
||||
careers_url: str,
|
||||
*,
|
||||
client: httpx.Client | None = None,
|
||||
) -> tuple[str | None, str]:
|
||||
"""Return (open_position_url, method) for a careers page. Never raises.
|
||||
|
||||
Tries ATS JSON → JSON-LD → job-like anchor → LLM classifier in order,
|
||||
returning on the first hit. method names which tier resolved it.
|
||||
"""
|
||||
_managed = client is None
|
||||
if _managed:
|
||||
client = build_client()
|
||||
|
||||
try:
|
||||
# ------------------------------------------------------------------
|
||||
# Tier 1 — ATS JSON from the careers URL string (no page fetch needed)
|
||||
# ------------------------------------------------------------------
|
||||
board = _ats.detect_ats_in_url(careers_url)
|
||||
if board is not None:
|
||||
try:
|
||||
pos_url = _fetch_ats_position(board, client)
|
||||
if pos_url:
|
||||
logger.info(
|
||||
"extract(%s): ats:%s from_url pos=%s",
|
||||
careers_url, board.ats_name, pos_url,
|
||||
)
|
||||
return pos_url, f"ats:{board.ats_name}"
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): ats url-tier error: %s", careers_url, exc)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Fetch page HTML — shared by all remaining tiers
|
||||
# ------------------------------------------------------------------
|
||||
html = _get_html(careers_url, client)
|
||||
if html is None:
|
||||
logger.info("extract(%s): page fetch failed; skipping remaining tiers", careers_url)
|
||||
return None, "none"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tier 1b — ATS JSON from page HTML (handles JS-embedded board pages)
|
||||
# ------------------------------------------------------------------
|
||||
html_board = _ats.detect_ats_in_html(html)
|
||||
if html_board is not None:
|
||||
try:
|
||||
pos_url2 = _fetch_ats_position(html_board, client)
|
||||
if pos_url2:
|
||||
logger.info(
|
||||
"extract(%s): ats:%s from_html pos=%s",
|
||||
careers_url, html_board.ats_name, pos_url2,
|
||||
)
|
||||
return pos_url2, f"ats:{html_board.ats_name}"
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): ats html-tier error: %s", careers_url, exc)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tier 2 — JobPosting JSON-LD
|
||||
# ------------------------------------------------------------------
|
||||
try:
|
||||
jld_url = _jsonld_job_url(html)
|
||||
if jld_url:
|
||||
logger.info("extract(%s): json_ld pos=%s", careers_url, jld_url)
|
||||
return jld_url, "json_ld"
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): json_ld tier error: %s", careers_url, exc)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tier 3 & 4 — Job-like anchors, then LLM (share one extraction pass)
|
||||
# ------------------------------------------------------------------
|
||||
try:
|
||||
anchors = extract_anchors(html, careers_url)
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): anchor extraction error: %s", careers_url, exc)
|
||||
anchors = []
|
||||
|
||||
# Tier 3 — first anchor whose URL *path* matches the job-URL pattern.
|
||||
# We search only the path component so hostnames like jobs.lever.co
|
||||
# don't trigger a false match (e.g. `//jobs` contains `/job`).
|
||||
try:
|
||||
for anchor in anchors:
|
||||
if _JOB_HREF_RE.search(urlsplit(anchor.url).path):
|
||||
logger.info("extract(%s): anchor pos=%s", careers_url, anchor.url)
|
||||
return anchor.url, "anchor"
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): anchor tier error: %s", careers_url, exc)
|
||||
|
||||
# Tier 4 — cheap-LLM job-link classifier (no-ops when LLM unconfigured)
|
||||
try:
|
||||
llm_result = classify_job_link(anchors)
|
||||
if llm_result is not None:
|
||||
logger.info("extract(%s): llm_classify pos=%s", careers_url, llm_result.job_url)
|
||||
return llm_result.job_url, "llm_classify"
|
||||
except Exception as exc:
|
||||
logger.warning("extract(%s): llm_classify tier error: %s", careers_url, exc)
|
||||
|
||||
logger.info("extract(%s): all tiers missed", careers_url)
|
||||
return None, "none"
|
||||
|
||||
finally:
|
||||
if _managed:
|
||||
client.close()
|
||||
|
||||
Reference in New Issue
Block a user