JobSourceAgent/jobsource/agent_fallback.py

"""Browser Use fused fallback: find careers page AND extract one job URL in one session.

This is the LAST tier of the cascade (Stage 2, Tier 6 / Stage 3, Tier 5). It fires only
when all cheaper tiers in cascade.py and extract.py have missed. A single Browser Use
agent session does both:
  1. Navigate to the company website and locate the careers/jobs page.
  2. From that page, open one specific open-position URL.

Both URLs are returned in one call — never two browser sessions for the same company.
A module-level memo keyed by normalized host ensures a same-domain second call (e.g.
cascade passes the root website, extract passes the careers URL on the same host) is a
cache hit with no new browser launch.

Graceful degradation:
  - ENABLE_BROWSER_AGENT=false       → log INFO, return None
  - Placeholder AGENT_MODEL/LLM key  → log INFO, return None
  - browser_use import error         → log WARNING, return None
  - Chromium/navigation/agent error  → log WARNING, return None
In all cases the batch still completes; those records land in needs_review.
"""
from __future__ import annotations

import asyncio
import glob
import logging
import os
import pathlib
import platform
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlsplit

from pydantic import BaseModel

from .config import get_settings

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Result models
# ---------------------------------------------------------------------------


class AgentFallbackResult(BaseModel):
    """Public result of find_and_extract()."""

    careers_url: str | None = None
    position_url: str | None = None


class _AgentOutput(BaseModel):
    """Structured-output schema handed to the Browser Use Agent."""

    careers_url: str | None = None
    position_url: str | None = None


# ---------------------------------------------------------------------------
# Module-level memo — one entry per normalized host
# ---------------------------------------------------------------------------

_MEMO: dict[str, AgentFallbackResult | None] = {}

# Max browser-agent steps before timing out.
_MAX_STEPS = 30

# Task prompt for the fused Stage-2 + Stage-3 session.
_TASK_TEMPLATE = (
    "You are helping a job-sourcing pipeline. Given a company website URL, do the following "
    "in a single browsing session:\n"
    "1. Navigate to {website}\n"
    "2. Find the company's official careers or jobs page (the page that lists open positions). "
    "   Explore navigation menus, footer links, or search for '/careers', '/jobs', etc.\n"
    "3. Once on the careers page, open one specific open-position detail page "
    "   (a page for exactly one role — not a listing index, filter, or category page).\n"
    "4. Return both URLs in the structured output fields:\n"
    "   - careers_url: the URL of the careers/jobs listing page\n"
    "   - position_url: the URL of one specific open-position detail page\n"
    "If you cannot find a careers page or any open positions, set the corresponding field to null."
)


# ---------------------------------------------------------------------------
# Availability gate
# ---------------------------------------------------------------------------


def _agent_unavailable() -> bool:
    """Return True when the browser agent is not configured (placeholder values)."""
    s = get_settings()
    return (
        s.agent_model.startswith("PLACEHOLDER")
        or s.llm_api_key.startswith("PLACEHOLDER")
    )


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _site_key(url: str) -> str:
    """Normalized netloc for memo keying — strips 'www.', lowercased."""
    host = urlsplit(url).netloc.lower()
    return host.removeprefix("www.")


def _find_chromium_executable() -> str | None:
    """Return the Playwright-installed Chromium binary path, or None if not found.

    browser-use 0.8.0's LocalBrowserWatchdog only globs `chrome-linux/chrome` but
    playwright ≥1.40 installs to `chrome-linux64/chrome`. Passing executable_path
    explicitly bypasses the watchdog's uvx-based auto-install fallback.
    """
    pw_base = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
    if not pw_base:
        system = platform.system()
        if system == "Darwin":
            pw_base = str(pathlib.Path.home() / "Library" / "Caches" / "ms-playwright")
        elif system == "Windows":
            pw_base = str(pathlib.Path.home() / "AppData" / "Local" / "ms-playwright")
        else:
            pw_base = str(pathlib.Path.home() / ".cache" / "ms-playwright")

    system = platform.system()
    if system == "Darwin":
        patterns = [
            f"{pw_base}/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium",
            f"{pw_base}/chromium-*/chrome-mac-arm64/Chromium.app/Contents/MacOS/Chromium",
        ]
    elif system == "Windows":
        patterns = [
            f"{pw_base}\\chromium-*\\chrome-win\\chrome.exe",
            f"{pw_base}\\chromium-*\\chrome-win64\\chrome.exe",
        ]
    else:
        patterns = [
            f"{pw_base}/chromium-*/chrome-linux64/chrome",
            f"{pw_base}/chromium-*/chrome-linux/chrome",
        ]

    for pattern in patterns:
        matches = sorted(glob.glob(pattern))
        if matches:
            return matches[-1]  # newest version last alphabetically

    return None


def _build_chat_model():  # type: ignore[return]
    """Construct the chat-model instance for the Browser Use agent.

    Test seam — monkeypatch this to inject a fake without touching Browser Use.
    Lazy-imports browser_use so a missing package never crashes module import.

    agent_model format: "provider:model_id" (e.g. "anthropic:claude-sonnet-4-6").
    Unknown/absent provider prefix defaults to ChatOpenAI.
    """
    from browser_use import ChatAnthropic, ChatOpenAI  # lazy import

    s = get_settings()
    model_str: str = s.agent_model
    api_key: str | None = None if s.llm_api_key.startswith("PLACEHOLDER") else s.llm_api_key

    if ":" in model_str:
        provider, model_id = model_str.split(":", 1)
    else:
        provider, model_id = "openai", model_str

    if provider == "anthropic":
        kwargs = {"model": model_id}
        if api_key:
            kwargs["api_key"] = api_key
        return ChatAnthropic(**kwargs)
    else:
        kwargs = {"model": model_id}
        if api_key:
            kwargs["openai_api_key"] = api_key
        return ChatOpenAI(**kwargs)


async def _agent_session(website: str, chat) -> _AgentOutput | None:  # type: ignore[return]
    """Run a single Browser Use session; return structured output or None."""
    from browser_use import Agent, Browser  # lazy import

    s = get_settings()
    # Pass executable_path explicitly so browser-use skips its uvx-based auto-installer.
    # browser-use 0.8.0's LocalBrowserWatchdog globs chrome-linux/chrome but playwright
    # ≥1.40 installs to chrome-linux64/chrome — the two paths diverge and the watchdog
    # falls through to `uvx playwright install` even when chromium is present.
    chromium_exe = _find_chromium_executable()
    browser = Browser(headless=s.browser_headless, executable_path=chromium_exe)
    task = _TASK_TEMPLATE.format(website=website)
    agent = Agent(
        task=task,
        llm=chat,
        browser=browser,
        output_model_schema=_AgentOutput,
    )
    try:
        history = await agent.run(max_steps=_MAX_STEPS)
        out = history.structured_output
        if isinstance(out, _AgentOutput):
            return out
        return None
    finally:
        try:
            await browser.close()
        except Exception:
            pass


def _run_blocking(coro_factory) -> _AgentOutput | None:  # type: ignore[return]
    """Run an async coroutine from synchronous code.

    Uses asyncio.run() when no event loop is running (the normal CLI path).
    Falls back to a one-shot ThreadPoolExecutor when called from inside an
    already-running loop (e.g. an async Prefect flow).
    """
    try:
        asyncio.get_running_loop()
        # Already inside a running loop — offload to a thread.
        with ThreadPoolExecutor(max_workers=1) as pool:
            future = pool.submit(asyncio.run, coro_factory())
            return future.result()
    except RuntimeError:
        # No running loop — safe to call asyncio.run() directly.
        return asyncio.run(coro_factory())


def _valid_url(val: str | None) -> str | None:
    """Return val if it's an http(s) URL, else None."""
    if val and isinstance(val, str) and val.startswith(("http://", "https://")):
        return val
    return None


# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------


def find_and_extract(website: str) -> AgentFallbackResult | None:
    """Fused Stage-2 + Stage-3 fallback: browser agent finds careers page AND one job URL.

    Returns an AgentFallbackResult on success, or None on any graceful failure.
    Never raises — all errors are logged and swallowed so the pipeline continues.
    Results are memoized per normalized host so a same-domain second call is free.
    """
    s = get_settings()

    if not s.enable_browser_agent:
        logger.info("agent_fallback(%s): ENABLE_BROWSER_AGENT=false, skipping", website)
        return None

    if _agent_unavailable():
        logger.info(
            "agent_fallback(%s): agent_model or llm_api_key not configured, skipping", website
        )
        return None

    key = _site_key(website)
    if key in _MEMO:
        logger.info("agent_fallback(%s): memo hit, returning cached result", website)
        return _MEMO[key]

    # Build the chat model (lazy import — handles missing browser_use gracefully).
    try:
        chat = _build_chat_model()
    except Exception as exc:
        logger.warning("agent_fallback(%s): failed to build chat model: %s", website, exc)
        _MEMO[key] = None
        return None

    # Run the fused browser session.
    try:
        raw: _AgentOutput | None = _run_blocking(lambda: _agent_session(website, chat))
    except Exception as exc:
        logger.warning("agent_fallback(%s): agent session error: %s", website, exc)
        _MEMO[key] = None
        return None

    if raw is None:
        logger.info("agent_fallback(%s): agent returned no structured output", website)
        _MEMO[key] = None
        return None

    result = AgentFallbackResult(
        careers_url=_valid_url(raw.careers_url),
        position_url=_valid_url(raw.position_url),
    )

    if result.careers_url or result.position_url:
        logger.info(
            "agent_fallback(%s): careers_url=%s position_url=%s",
            website, result.careers_url, result.position_url,
        )
    else:
        logger.info("agent_fallback(%s): agent found no usable URLs", website)

    _MEMO[key] = result
    return result