CGA Recruiting App Jobs API.

from urllib.parse import urlparse, parse_qs

SUFFIXES = (".my.site.com", ".my.salesforce-sites.com")
PAGES = {"cga__joblisting", "cga__jobdetails"}

def parse_cga(url: str) -> dict | None:
    parsed = urlparse(url)
    host = parsed.netloc.lower()
    if parsed.scheme != "https" or not host.endswith(SUFFIXES):
        return None

    path = parsed.path
    slash = path.rfind("/")
    page = path[slash + 1:] if slash >= 0 else path.lstrip("/")
    if page.lower() not in PAGES:
        return None

    site_path = path[:slash] if slash > 0 else ""
    query = {k.lower(): v[0] for k, v in parse_qs(parsed.query).items()}
    return {
        "org": host.split(".")[0],
        "site_path": site_path,
        "board_url": f"https://{host}{site_path}/cga__JobListing",
        "job_number": query.get("jobnumber"),
    }

print(parse_cga("https://kolekto.my.salesforce-sites.com/jobs/cga__JobDetails"
                "?q=&jobNumber=JOB00440"))

import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

ARTIFACTS = re.compile("cga__(?:RecruitingAppAssets|JobListing)", re.IGNORECASE)

def parse_listings(html: str, page_url: str) -> list[dict]:
    if not ARTIFACTS.search(html):
        raise RuntimeError("page did not contain CGA Recruiting artifacts")

    soup = BeautifulSoup(html, "html.parser")
    rows = []
    selector = "a.searchResultLink[href*='cga__JobDetails'][href*='jobNumber=']"
    for anchor in soup.select(selector):
        detail_url = urljoin(page_url, anchor["href"])
        identity = parse_cga(detail_url)
        title = anchor.get_text(strip=True)
        if not identity or not identity["job_number"] or not title:
            continue
        rows.append({
            "id": identity["job_number"],
            "title": title,
            "url": detail_url,
        })
    return rows

session = requests.Session()
board_url = "https://compassgovernmentsolutions.my.site.com/cga__JobListing"
first = session.get(board_url, timeout=30)
first.raise_for_status()
listings = parse_listings(first.text, board_url)

SOURCE_PARAM = re.compile(
    "'parameters'[ ]*:[ ]*[{][ ]*'([^']+)'[ ]*:[ ]*'([^']+)'", re.IGNORECASE)

def build_next_request(html: str, page_url: str) -> dict | None:
    soup = BeautifulSoup(html, "html.parser")
    button = soup.select_one("input[title='Next']:not([disabled])")
    if not button or not button.get("onclick"):
        return None

    match = SOURCE_PARAM.search(button["onclick"])
    form = button.find_parent("form")
    if not match or form is None:
        return None

    fields = {}
    for node in form.select("input[name]"):
        if node.get("type") in ("checkbox", "radio") and not node.has_attr("checked"):
            continue
        fields[node["name"]] = node.get("value") or ""
    fields[match.group(1)] = match.group(2)

    action = form.get("action") or page_url
    return {"url": urljoin(page_url, action), "fields": fields}

def scrape_board(session, board_url: str, max_pages: int = 20) -> list[dict]:
    resp = session.get(board_url, timeout=30)
    resp.raise_for_status()
    html, page_url = resp.text, board_url
    jobs, seen = [], set()

    for _ in range(max_pages):
        for row in parse_listings(html, page_url):
            if row["id"] not in seen:
                seen.add(row["id"])
                jobs.append(row)

        nxt = build_next_request(html, page_url)
        if not nxt:
            break
        resp = session.post(nxt["url"], data=nxt["fields"],
                            headers={"Referer": board_url}, timeout=30)
        resp.raise_for_status()
        html, page_url = resp.text, nxt["url"]
    return jobs

import json

def find_job_posting(html: str) -> dict | None:
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup.find_all("script", type="application/ld+json"):
        try:
            data = json.loads(tag.string or "")
        except (json.JSONDecodeError, TypeError):
            continue
        for node in (data if isinstance(data, list) else [data]):
            if isinstance(node, dict) and node.get("@type") == "JobPosting":
                return node
    return None

APPLY_TARGET = re.compile("window[.]location[ ]*=[ ]*'([^']+)'", re.IGNORECASE)

def fetch_detail(session, listing: dict) -> dict | None:
    resp = session.get(listing["url"], timeout=30)
    if resp.status_code in (404, 410):
        return None
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text, "html.parser")
    posting = find_job_posting(resp.text) or {}
    blocks = [b.decode_contents().strip()
              for b in soup.select(".htmlDetailElementTable .sfdc_richtext")
              if len(b.get_text(strip=True)) >= 30]
    description = "".join(blocks) or posting.get("description")

    address = ((posting.get("jobLocation") or {}).get("address")) or {}
    apply_button = soup.select_one("input.applyButton[onclick]")
    apply_match = APPLY_TARGET.search(apply_button["onclick"]) if apply_button else None

    return {
        "id": listing["id"],
        "title": posting.get("title") or listing["title"],
        "description_html": description,
        "city": address.get("addressLocality"),
        "state": address.get("addressRegion"),
        "country": address.get("addressCountry"),
        "url": listing["url"],
        "apply_url": (urljoin(listing["url"], apply_match.group(1))
                      if apply_match else listing["url"]),
    }