GovernmentJobs (NeoGov) Jobs API.

from urllib.parse import urlparse

HOST = "www.governmentjobs.com"
RESERVED = {"home", "agencycustomstylescheme"}

def extract_agency_slug(url: str) -> str | None:
    parsed = urlparse(url)
    if parsed.netloc.lower() != HOST:
        return None
    parts = [p for p in parsed.path.strip("/").split("/") if p]
    # Boards live at /careers/{slug}[/...]; the first segment must be "careers".
    if len(parts) < 2 or parts[0].lower() != "careers":
        return None
    slug = parts[1]
    return None if slug.lower() in RESERVED else slug

# "https://www.governmentjobs.com/careers/seattle/jobs/5338985" -> "seattle"

import requests
from bs4 import BeautifulSoup

BASE = f"https://{HOST}"

def listings_url(slug: str, page: int) -> str:
    return (
        f"{BASE}/careers/home/index"
        f"?agency={slug}&page={page}&sort=PositionTitle&isDescendingSort=false"
    )

def has_next_page(soup: BeautifulSoup) -> bool:
    nxt = soup.select_one("li.PagedList-skipToNext")
    return nxt is not None and "disabled" not in nxt.get("class", [])

def fetch_listings(slug: str, session: requests.Session, max_pages: int = 100) -> list[dict]:
    jobs, seen = [], set()
    page = 1
    while page <= max_pages:
        resp = session.get(
            listings_url(slug, page),
            headers={
                "X-Requested-With": "XMLHttpRequest",  # fragment only served for XHR
                "Referer": f"{BASE}/careers/{slug}",
            },
            timeout=30,
        )
        resp.raise_for_status()
        soup = BeautifulSoup(resp.text, "html.parser")

        added = 0
        for card in soup.select("li.list-item[data-job-id]"):
            job_id = (card.get("data-job-id") or "").strip()
            if not job_id or job_id in seen:
                continue
            seen.add(job_id)
            added += 1
            jobs.append(parse_card(card, slug, job_id))

        # Defensive stop: no new IDs on this page, or no enabled "next" control.
        if added == 0 or not has_next_page(soup):
            break
        page += 1
    return jobs

def parse_card(card, slug: str, job_id: str) -> dict:
    link = card.select_one("a.item-details-link")
    href = (link.get("href") if link else None) or f"/careers/{slug}/jobs/{job_id}"
    detail_url = requests.compat.urljoin(f"{BASE}/", href)

    department = category = salary_summary = location = None
    for li in card.select("ul.list-meta > li"):
        text = " ".join(li.get_text(" ", strip=True).split())
        if not text:
            continue
        classes = li.get("class", [])
        if "categories-list" in classes or text.startswith("Category:"):
            category = text.removeprefix("Category:").strip()
        elif text.startswith("Department:"):
            department = text.removeprefix("Department:").strip()
        elif "Full-Time" in text or "Part-Time" in text or "$" in text:
            salary_summary = text
        elif location is None:
            location = text

    return {
        "id": job_id,
        "title": link.get_text(strip=True) if link else None,
        "listing_url": detail_url,
        "department": department,
        "category": category,
        "salary_summary": salary_summary,
        "location": location,
    }

import html as html_lib
import json

def find_job_posting(soup: BeautifulSoup) -> dict | None:
    for tag in soup.find_all("script", type="application/ld+json"):
        try:
            data = json.loads(tag.string or "")
        except (json.JSONDecodeError, TypeError):
            continue
        for node in (data if isinstance(data, list) else [data]):
            if isinstance(node, dict) and node.get("@type") == "JobPosting":
                return node
    return None

def fetch_detail(detail_url: str, session: requests.Session) -> dict:
    resp = session.get(detail_url, timeout=30)
    resp.raise_for_status()
    soup = BeautifulSoup(resp.text, "html.parser")

    posting = find_job_posting(soup)
    if not posting:
        # Fallback: the description is also rendered under #details-info.
        node = soup.select_one("#details-info")
        return {"description_html": node.decode_contents() if node else None}

    salary = (posting.get("baseSalary") or {}).get("value") or {}
    raw_desc = posting.get("description")
    return {
        "title": posting.get("title"),
        # GovernmentJobs double-encodes the description (<p>...), so unescape it.
        "description_html": html_lib.unescape(raw_desc) if raw_desc else None,
        "posted_at": posting.get("datePosted"),
        "closes_at": posting.get("validThrough"),
        "employment_type": posting.get("employmentType"),
        "salary_min": salary.get("minValue"),
        "salary_max": salary.get("maxValue"),
        "salary_unit": salary.get("unitText"),
        "organization": (posting.get("hiringOrganization") or {}).get("name"),
    }