233 lines
9.5 KiB
Python
233 lines
9.5 KiB
Python
"""Extract one open position URL from a careers page (Stage 3).
|
|
|
|
Cascade order (return early on first hit):
|
|
1. ATS JSON (URL) — detect ATS board coordinates in the careers URL itself;
|
|
call the board's public JSON API for the first posting URL.
|
|
1b. ATS JSON (HTML) — fetch the careers page; detect ATS board in its HTML;
|
|
call the board API. Handles inline JS embeds.
|
|
2. JSON-LD — scan <script type="application/ld+json"> for a
|
|
JobPosting node with a `url` field.
|
|
3. Job-like anchors — first <a> whose href matches /job|/position|/opening|/vacanc.
|
|
4. Cheap-LLM — classify_job_link() from classify_llm; no-ops gracefully
|
|
when the LLM is not configured.
|
|
5. Browser agent — agent_fallback.find_and_extract(); fused with Stage-2 so
|
|
the same memoized session covers both stages.
|
|
6. Miss — return (None, "none").
|
|
|
|
Returns (url: str | None, method: str). Never raises — any tier failure logs
|
|
and falls through. method is one of:
|
|
"ats:{name}" | "json_ld" | "anchor" | "llm_classify" | "browser_agent" | "none"
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from urllib.parse import urlsplit
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from . import agent_fallback as _agent_fallback
|
|
from .careers import ats as _ats
|
|
from .careers.classify_llm import classify_job_link, extract_anchors
|
|
from .config import get_settings
|
|
from .http import build_client, request_with_retries
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Matches href paths that indicate a single-job detail page.
|
|
_JOB_HREF_RE = re.compile(r"/(job|position|opening|vacanc)", re.IGNORECASE)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _get_html(url: str, client: httpx.Client) -> str | None:
|
|
"""Fetch page HTML; return text on success or None on any error/non-2xx."""
|
|
try:
|
|
resp = request_with_retries(client, "GET", url, max_retries=1)
|
|
if resp.status_code < 400:
|
|
return resp.text
|
|
logger.info("extract._get_html(%s): HTTP %s", url, resp.status_code)
|
|
return None
|
|
except Exception as exc:
|
|
logger.info("extract._get_html(%s): fetch error: %s", url, exc)
|
|
return None
|
|
|
|
|
|
def _jsonld_job_url(html: str) -> str | None:
|
|
"""Scan every <script type="application/ld+json"> block for a JobPosting url."""
|
|
soup = BeautifulSoup(html, "lxml")
|
|
for script in soup.find_all("script", type="application/ld+json"):
|
|
try:
|
|
data = json.loads(script.get_text(strip=True))
|
|
except (json.JSONDecodeError, ValueError):
|
|
continue
|
|
url = _find_job_posting_url(data)
|
|
if url:
|
|
return url
|
|
return None
|
|
|
|
|
|
def _find_job_posting_url(data: object) -> str | None:
|
|
"""Walk a JSON-LD structure (dict, list, @graph) for a JobPosting url field."""
|
|
if isinstance(data, list):
|
|
for item in data:
|
|
result = _find_job_posting_url(item)
|
|
if result:
|
|
return result
|
|
return None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
# Expand @graph (JSON-LD named graph — a list of typed nodes)
|
|
graph = data.get("@graph")
|
|
if graph:
|
|
result = _find_job_posting_url(graph)
|
|
if result:
|
|
return result
|
|
# Check this node's @type
|
|
type_val = data.get("@type", "")
|
|
if isinstance(type_val, list):
|
|
is_job = "JobPosting" in type_val
|
|
else:
|
|
is_job = type_val == "JobPosting"
|
|
if is_job:
|
|
url = data.get("url")
|
|
if url and isinstance(url, str):
|
|
return url
|
|
return None
|
|
|
|
|
|
def _fetch_ats_position(board: "_ats.ATSBoard", client: httpx.Client) -> str | None:
|
|
"""Call the ATS public JSON API for *board* and return the first job URL or None."""
|
|
fetch_fn = _ats._FETCH_DISPATCH.get(board.ats_name)
|
|
if fetch_fn is None:
|
|
return None
|
|
fetch = fetch_fn(board, client) # type: ignore[operator]
|
|
return fetch.first_url
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_open_position(
|
|
careers_url: str,
|
|
*,
|
|
client: httpx.Client | None = None,
|
|
) -> tuple[str | None, str]:
|
|
"""Return (open_position_url, method) for a careers page. Never raises.
|
|
|
|
Tries ATS JSON → JSON-LD → job-like anchor → LLM classifier in order,
|
|
returning on the first hit. method names which tier resolved it.
|
|
"""
|
|
_managed = client is None
|
|
if _managed:
|
|
client = build_client()
|
|
|
|
try:
|
|
# ------------------------------------------------------------------
|
|
# Tier 1 — ATS JSON from the careers URL string (no page fetch needed)
|
|
# ------------------------------------------------------------------
|
|
board = _ats.detect_ats_in_url(careers_url)
|
|
if board is not None:
|
|
try:
|
|
pos_url = _fetch_ats_position(board, client)
|
|
if pos_url:
|
|
logger.info(
|
|
"extract(%s): ats:%s from_url pos=%s",
|
|
careers_url, board.ats_name, pos_url,
|
|
)
|
|
return pos_url, f"ats:{board.ats_name}"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): ats url-tier error: %s", careers_url, exc)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Fetch page HTML — shared by all remaining tiers
|
|
# ------------------------------------------------------------------
|
|
html = _get_html(careers_url, client)
|
|
if html is None:
|
|
logger.info("extract(%s): page fetch failed; skipping remaining tiers", careers_url)
|
|
return None, "none"
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tier 1b — ATS JSON from page HTML (handles JS-embedded board pages)
|
|
# ------------------------------------------------------------------
|
|
html_board = _ats.detect_ats_in_html(html)
|
|
if html_board is not None:
|
|
try:
|
|
pos_url2 = _fetch_ats_position(html_board, client)
|
|
if pos_url2:
|
|
logger.info(
|
|
"extract(%s): ats:%s from_html pos=%s",
|
|
careers_url, html_board.ats_name, pos_url2,
|
|
)
|
|
return pos_url2, f"ats:{html_board.ats_name}"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): ats html-tier error: %s", careers_url, exc)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tier 2 — JobPosting JSON-LD
|
|
# ------------------------------------------------------------------
|
|
try:
|
|
jld_url = _jsonld_job_url(html)
|
|
if jld_url:
|
|
logger.info("extract(%s): json_ld pos=%s", careers_url, jld_url)
|
|
return jld_url, "json_ld"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): json_ld tier error: %s", careers_url, exc)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tier 3 & 4 — Job-like anchors, then LLM (share one extraction pass)
|
|
# ------------------------------------------------------------------
|
|
try:
|
|
anchors = extract_anchors(html, careers_url)
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): anchor extraction error: %s", careers_url, exc)
|
|
anchors = []
|
|
|
|
# Tier 3 — first anchor whose URL *path* matches the job-URL pattern.
|
|
# We search only the path component so hostnames like jobs.lever.co
|
|
# don't trigger a false match (e.g. `//jobs` contains `/job`).
|
|
try:
|
|
for anchor in anchors:
|
|
if _JOB_HREF_RE.search(urlsplit(anchor.url).path):
|
|
logger.info("extract(%s): anchor pos=%s", careers_url, anchor.url)
|
|
return anchor.url, "anchor"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): anchor tier error: %s", careers_url, exc)
|
|
|
|
# Tier 4 — cheap-LLM job-link classifier (no-ops when LLM unconfigured)
|
|
try:
|
|
llm_result = classify_job_link(anchors)
|
|
if llm_result is not None:
|
|
logger.info("extract(%s): llm_classify pos=%s", careers_url, llm_result.job_url)
|
|
return llm_result.job_url, "llm_classify"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): llm_classify tier error: %s", careers_url, exc)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tier 5 — Browser agent (fused; memo-shared with cascade Tier 6)
|
|
# ------------------------------------------------------------------
|
|
if get_settings().enable_browser_agent:
|
|
try:
|
|
ag = _agent_fallback.find_and_extract(careers_url)
|
|
if ag and ag.position_url:
|
|
logger.info(
|
|
"extract(%s): browser_agent pos=%s", careers_url, ag.position_url
|
|
)
|
|
return ag.position_url, "browser_agent"
|
|
except Exception as exc:
|
|
logger.warning("extract(%s): browser_agent tier error: %s", careers_url, exc)
|
|
|
|
logger.info("extract(%s): all tiers missed", careers_url)
|
|
return None, "none"
|
|
|
|
finally:
|
|
if _managed:
|
|
client.close()
|