agent-fallback
This commit is contained in:
@@ -1,11 +1,301 @@
|
||||
"""Browser Use fused fallback: find careers page AND extract one job URL in one session.
|
||||
|
||||
Scaffold stub -- not implemented yet.
|
||||
This is the LAST tier of the cascade (Stage 2, Tier 6 / Stage 3, Tier 5). It fires only
|
||||
when all cheaper tiers in cascade.py and extract.py have missed. A single Browser Use
|
||||
agent session does both:
|
||||
1. Navigate to the company website and locate the careers/jobs page.
|
||||
2. From that page, open one specific open-position URL.
|
||||
|
||||
Both URLs are returned in one call — never two browser sessions for the same company.
|
||||
A module-level memo keyed by normalized host ensures a same-domain second call (e.g.
|
||||
cascade passes the root website, extract passes the careers URL on the same host) is a
|
||||
cache hit with no new browser launch.
|
||||
|
||||
Graceful degradation:
|
||||
- ENABLE_BROWSER_AGENT=false → log INFO, return None
|
||||
- Placeholder AGENT_MODEL/LLM key → log INFO, return None
|
||||
- browser_use import error → log WARNING, return None
|
||||
- Chromium/navigation/agent error → log WARNING, return None
|
||||
In all cases the batch still completes; those records land in needs_review.
|
||||
"""
|
||||
# TODO (Stage 2/3 last resort): implement per CLAUDE.md "Stage 2 — tier 6" and "Stage 3 — tier 5".
|
||||
# This is the LAST tier of the cascade. Fires only when all cheaper tiers in cascade.py
|
||||
# and extract.py have failed. One Browser Use agent session does both:
|
||||
# 1. Navigate to the company website and locate the careers/jobs page.
|
||||
# 2. From the careers page, return the URL of one open position.
|
||||
# Graceful degradation: if Browser Use / Playwright / LLM key are unavailable, log clearly
|
||||
# and return (careers_url=None, position_url=None) so the pipeline records needs_review.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AgentFallbackResult(BaseModel):
|
||||
"""Public result of find_and_extract()."""
|
||||
|
||||
careers_url: str | None = None
|
||||
position_url: str | None = None
|
||||
|
||||
|
||||
class _AgentOutput(BaseModel):
|
||||
"""Structured-output schema handed to the Browser Use Agent."""
|
||||
|
||||
careers_url: str | None = None
|
||||
position_url: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level memo — one entry per normalized host
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MEMO: dict[str, AgentFallbackResult | None] = {}
|
||||
|
||||
# Max browser-agent steps before timing out.
|
||||
_MAX_STEPS = 30
|
||||
|
||||
# Task prompt for the fused Stage-2 + Stage-3 session.
|
||||
_TASK_TEMPLATE = (
|
||||
"You are helping a job-sourcing pipeline. Given a company website URL, do the following "
|
||||
"in a single browsing session:\n"
|
||||
"1. Navigate to {website}\n"
|
||||
"2. Find the company's official careers or jobs page (the page that lists open positions). "
|
||||
" Explore navigation menus, footer links, or search for '/careers', '/jobs', etc.\n"
|
||||
"3. Once on the careers page, open one specific open-position detail page "
|
||||
" (a page for exactly one role — not a listing index, filter, or category page).\n"
|
||||
"4. Return both URLs in the structured output fields:\n"
|
||||
" - careers_url: the URL of the careers/jobs listing page\n"
|
||||
" - position_url: the URL of one specific open-position detail page\n"
|
||||
"If you cannot find a careers page or any open positions, set the corresponding field to null."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Availability gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _agent_unavailable() -> bool:
|
||||
"""Return True when the browser agent is not configured (placeholder values)."""
|
||||
s = get_settings()
|
||||
return (
|
||||
s.agent_model.startswith("PLACEHOLDER")
|
||||
or s.llm_api_key.startswith("PLACEHOLDER")
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _site_key(url: str) -> str:
|
||||
"""Normalized netloc for memo keying — strips 'www.', lowercased."""
|
||||
host = urlsplit(url).netloc.lower()
|
||||
return host.removeprefix("www.")
|
||||
|
||||
|
||||
def _find_chromium_executable() -> str | None:
|
||||
"""Return the Playwright-installed Chromium binary path, or None if not found.
|
||||
|
||||
browser-use 0.8.0's LocalBrowserWatchdog only globs `chrome-linux/chrome` but
|
||||
playwright ≥1.40 installs to `chrome-linux64/chrome`. Passing executable_path
|
||||
explicitly bypasses the watchdog's uvx-based auto-install fallback.
|
||||
"""
|
||||
pw_base = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
|
||||
if not pw_base:
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
pw_base = str(pathlib.Path.home() / "Library" / "Caches" / "ms-playwright")
|
||||
elif system == "Windows":
|
||||
pw_base = str(pathlib.Path.home() / "AppData" / "Local" / "ms-playwright")
|
||||
else:
|
||||
pw_base = str(pathlib.Path.home() / ".cache" / "ms-playwright")
|
||||
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
patterns = [
|
||||
f"{pw_base}/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium",
|
||||
f"{pw_base}/chromium-*/chrome-mac-arm64/Chromium.app/Contents/MacOS/Chromium",
|
||||
]
|
||||
elif system == "Windows":
|
||||
patterns = [
|
||||
f"{pw_base}\\chromium-*\\chrome-win\\chrome.exe",
|
||||
f"{pw_base}\\chromium-*\\chrome-win64\\chrome.exe",
|
||||
]
|
||||
else:
|
||||
patterns = [
|
||||
f"{pw_base}/chromium-*/chrome-linux64/chrome",
|
||||
f"{pw_base}/chromium-*/chrome-linux/chrome",
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
matches = sorted(glob.glob(pattern))
|
||||
if matches:
|
||||
return matches[-1] # newest version last alphabetically
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _build_chat_model(): # type: ignore[return]
|
||||
"""Construct the chat-model instance for the Browser Use agent.
|
||||
|
||||
Test seam — monkeypatch this to inject a fake without touching Browser Use.
|
||||
Lazy-imports browser_use so a missing package never crashes module import.
|
||||
|
||||
agent_model format: "provider:model_id" (e.g. "anthropic:claude-sonnet-4-6").
|
||||
Unknown/absent provider prefix defaults to ChatOpenAI.
|
||||
"""
|
||||
from browser_use import ChatAnthropic, ChatOpenAI # lazy import
|
||||
|
||||
s = get_settings()
|
||||
model_str: str = s.agent_model
|
||||
api_key: str | None = None if s.llm_api_key.startswith("PLACEHOLDER") else s.llm_api_key
|
||||
|
||||
if ":" in model_str:
|
||||
provider, model_id = model_str.split(":", 1)
|
||||
else:
|
||||
provider, model_id = "openai", model_str
|
||||
|
||||
if provider == "anthropic":
|
||||
kwargs = {"model": model_id}
|
||||
if api_key:
|
||||
kwargs["api_key"] = api_key
|
||||
return ChatAnthropic(**kwargs)
|
||||
else:
|
||||
kwargs = {"model": model_id}
|
||||
if api_key:
|
||||
kwargs["openai_api_key"] = api_key
|
||||
return ChatOpenAI(**kwargs)
|
||||
|
||||
|
||||
async def _agent_session(website: str, chat) -> _AgentOutput | None: # type: ignore[return]
|
||||
"""Run a single Browser Use session; return structured output or None."""
|
||||
from browser_use import Agent, Browser # lazy import
|
||||
|
||||
s = get_settings()
|
||||
# Pass executable_path explicitly so browser-use skips its uvx-based auto-installer.
|
||||
# browser-use 0.8.0's LocalBrowserWatchdog globs chrome-linux/chrome but playwright
|
||||
# ≥1.40 installs to chrome-linux64/chrome — the two paths diverge and the watchdog
|
||||
# falls through to `uvx playwright install` even when chromium is present.
|
||||
chromium_exe = _find_chromium_executable()
|
||||
browser = Browser(headless=s.browser_headless, executable_path=chromium_exe)
|
||||
task = _TASK_TEMPLATE.format(website=website)
|
||||
agent = Agent(
|
||||
task=task,
|
||||
llm=chat,
|
||||
browser=browser,
|
||||
output_model_schema=_AgentOutput,
|
||||
)
|
||||
try:
|
||||
history = await agent.run(max_steps=_MAX_STEPS)
|
||||
out = history.structured_output
|
||||
if isinstance(out, _AgentOutput):
|
||||
return out
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
await browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _run_blocking(coro_factory) -> _AgentOutput | None: # type: ignore[return]
|
||||
"""Run an async coroutine from synchronous code.
|
||||
|
||||
Uses asyncio.run() when no event loop is running (the normal CLI path).
|
||||
Falls back to a one-shot ThreadPoolExecutor when called from inside an
|
||||
already-running loop (e.g. an async Prefect flow).
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
# Already inside a running loop — offload to a thread.
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(asyncio.run, coro_factory())
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
# No running loop — safe to call asyncio.run() directly.
|
||||
return asyncio.run(coro_factory())
|
||||
|
||||
|
||||
def _valid_url(val: str | None) -> str | None:
|
||||
"""Return val if it's an http(s) URL, else None."""
|
||||
if val and isinstance(val, str) and val.startswith(("http://", "https://")):
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_and_extract(website: str) -> AgentFallbackResult | None:
|
||||
"""Fused Stage-2 + Stage-3 fallback: browser agent finds careers page AND one job URL.
|
||||
|
||||
Returns an AgentFallbackResult on success, or None on any graceful failure.
|
||||
Never raises — all errors are logged and swallowed so the pipeline continues.
|
||||
Results are memoized per normalized host so a same-domain second call is free.
|
||||
"""
|
||||
s = get_settings()
|
||||
|
||||
if not s.enable_browser_agent:
|
||||
logger.info("agent_fallback(%s): ENABLE_BROWSER_AGENT=false, skipping", website)
|
||||
return None
|
||||
|
||||
if _agent_unavailable():
|
||||
logger.info(
|
||||
"agent_fallback(%s): agent_model or llm_api_key not configured, skipping", website
|
||||
)
|
||||
return None
|
||||
|
||||
key = _site_key(website)
|
||||
if key in _MEMO:
|
||||
logger.info("agent_fallback(%s): memo hit, returning cached result", website)
|
||||
return _MEMO[key]
|
||||
|
||||
# Build the chat model (lazy import — handles missing browser_use gracefully).
|
||||
try:
|
||||
chat = _build_chat_model()
|
||||
except Exception as exc:
|
||||
logger.warning("agent_fallback(%s): failed to build chat model: %s", website, exc)
|
||||
_MEMO[key] = None
|
||||
return None
|
||||
|
||||
# Run the fused browser session.
|
||||
try:
|
||||
raw: _AgentOutput | None = _run_blocking(lambda: _agent_session(website, chat))
|
||||
except Exception as exc:
|
||||
logger.warning("agent_fallback(%s): agent session error: %s", website, exc)
|
||||
_MEMO[key] = None
|
||||
return None
|
||||
|
||||
if raw is None:
|
||||
logger.info("agent_fallback(%s): agent returned no structured output", website)
|
||||
_MEMO[key] = None
|
||||
return None
|
||||
|
||||
result = AgentFallbackResult(
|
||||
careers_url=_valid_url(raw.careers_url),
|
||||
position_url=_valid_url(raw.position_url),
|
||||
)
|
||||
|
||||
if result.careers_url or result.position_url:
|
||||
logger.info(
|
||||
"agent_fallback(%s): careers_url=%s position_url=%s",
|
||||
website, result.careers_url, result.position_url,
|
||||
)
|
||||
else:
|
||||
logger.info("agent_fallback(%s): agent found no usable URLs", website)
|
||||
|
||||
_MEMO[key] = result
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user