FCMS Candidate Portal Jobs API.

from urllib.parse import urlparse, parse_qs, quote

SUFFIX = ".my.salesforce-sites.com"
PAGE = "fcms__cmslayout"

def parse_fcms(url: str) -> dict | None:
    parsed = urlparse(url)
    host = parsed.netloc.lower()
    if parsed.scheme != "https" or not host.endswith(SUFFIX):
        return None

    path = parsed.path
    slash = path.rfind("/")
    page_name = path[slash + 1:] if slash >= 0 else path.lstrip("/")
    if page_name.lower() != PAGE:
        return None

    query = {k.lower(): v[0] for k, v in parse_qs(parsed.query).items()}
    view = query.get("page") or ""
    # Only the list and detail views are boards or jobs; login pages are not.
    if view and not (view.endswith("ListPage") or view.endswith("DetailPage")):
        return None

    site_path = path[:slash] if slash > 0 else ""
    job_site = query.get("jobsite") or "NULL"
    board = (f"https://{host}{site_path}/FCMS__CMSLayout"
             f"?page=JobListPage&p=Candidate&jobSite={quote(job_site)}")
    return {
        "org": host.split(".")[0],
        "site_path": site_path,
        "job_site": job_site,
        "board_url": board,
        "job_id": query.get("jobids"),
    }

print(parse_fcms("https://njtransit.my.salesforce-sites.com/FCMS__CMSLayout"
                 "?page=JobListPage&p=Candidate&jobSite=NULL"))

import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

ARTIFACTS = re.compile("FCMS__(?:CMSLayout|CMSSiteLogin|bootStrapFiles)",
                       re.IGNORECASE)

def same_origin(left: str, right: str) -> bool:
    a, b = urlparse(left), urlparse(right)
    return (a.scheme, a.netloc.lower()) == (b.scheme, b.netloc.lower())

def find_frame(session, shell_url: str, kind: str) -> str:
    resp = session.get(shell_url, timeout=30)
    resp.raise_for_status()
    if not ARTIFACTS.search(resp.text):
        raise RuntimeError("page did not contain FCMS package artifacts")

    soup = BeautifulSoup(resp.text, "html.parser")
    for frame in soup.select("iframe[src]"):
        src = frame.get("src") or ""
        if f"Job{kind}" not in src:
            continue
        resolved = urljoin(shell_url, src)
        if same_origin(shell_url, resolved):
            return resolved
    raise RuntimeError(f"FCMS shell exposed no same-origin Job{kind} frame")

session = requests.Session()
board = "https://njtransit.my.salesforce-sites.com/FCMS__CMSLayout?page=JobListPage&p=Candidate&jobSite=NULL"
frame_url = find_frame(session, board, "List")

def build_headers(table) -> dict:
    headers = {}
    if table is None:
        return headers
    cells = table.select("thead th") or table.select("tr:first-child th")
    for index, cell in enumerate(cells):
        key = re.sub("[^a-z0-9]", "", cell.get_text(strip=True).lower())
        if key:
            headers.setdefault(key, index)
    return headers

def read_cell(cells, headers, names) -> str | None:
    for key, index in headers.items():
        if any(name in key for name in names) and index < len(cells):
            return cells[index].get_text(strip=True)
    return None

def parse_rows(html: str, frame_url: str) -> list[dict]:
    soup = BeautifulSoup(html, "html.parser")
    rows = []
    for anchor in soup.select("a[href*='FCMS__CMSLayout'][href*='jobIds=']"):
        detail_url = urljoin(frame_url, anchor["href"])
        identity = parse_fcms(detail_url)
        title = anchor.get_text(strip=True)
        if not identity or not identity["job_id"] or not title:
            continue

        row = anchor.find_parent("tr")
        cells = row.select("td") if row else []
        headers = build_headers(row.find_parent("table") if row else None)
        city = read_cell(cells, headers, ["city"])
        state = read_cell(cells, headers, ["state", "region"])
        rows.append({
            "id": identity["job_id"],
            "title": title,
            "url": detail_url,
            "city": city,
            "state": state,
            "department": read_cell(cells, headers, ["department", "division"]),
            "employment_type": read_cell(cells, headers, ["term", "type"]),
        })
    return rows

import json
import time

JSF_POSTBACK = re.compile(
    "jsfcljs[(]document[.]getElementById[(]'([^']+)'[)],'([^']+)'", re.IGNORECASE)

def build_next(html: str, frame_url: str) -> dict | None:
    soup = BeautifulSoup(html, "html.parser")
    control = None
    for anchor in soup.select("a[onclick]"):
        if anchor.get_text(strip=True).lower() == "next":
            control = anchor
            break
    if control is None:
        return None

    match = JSF_POSTBACK.search(control.get("onclick") or "")
    if not match:
        return None
    form = soup.find(id=match.group(1))
    if form is None:
        return None

    fields = {}
    for node in form.select("input[name]"):
        if node.get("type") in ("checkbox", "radio") and not node.has_attr("checked"):
            continue
        fields[node["name"]] = node.get("value") or ""

    # Visualforce keeps its signed postback state OUTSIDE the job-list form.
    for node in soup.select("input[name^='com.salesforce.visualforce.']"):
        fields[node["name"]] = node.get("value") or ""

    pairs = match.group(2).split(",")
    for index in range(0, len(pairs) - 1, 2):
        fields[pairs[index]] = pairs[index + 1]

    action = form.get("action") or frame_url
    return {"url": urljoin(frame_url, action), "fields": fields}

def scrape_board(session, board_url: str, max_pages: int = 20) -> list[dict]:
    frame_url = find_frame(session, board_url, "List")
    resp = session.get(frame_url, timeout=30)
    resp.raise_for_status()
    html, jobs, seen = resp.text, [], set()

    for _ in range(max_pages):
        for row in parse_rows(html, frame_url):
            if row["id"] not in seen:
                seen.add(row["id"])
                jobs.append(row)

        nxt = build_next(html, frame_url)
        if not nxt:
            break
        resp = session.post(
            nxt["url"], data=nxt["fields"],
            headers={"Referer": frame_url,
                     "Origin": urlparse(frame_url).scheme + "://" + urlparse(frame_url).netloc},
            timeout=30)
        resp.raise_for_status()
        html, frame_url = resp.text, nxt["url"]
        time.sleep(0.3)
    return jobs

APPLY_TARGET = re.compile("window[.]open[(]'([^']*page=JobApplicationPage[^']*)'",
                          re.IGNORECASE)

def match_label(text: str, label: str) -> str | None:
    match = re.search(label + "[ ]*:[ ]*([^\r\n]+)", text, re.IGNORECASE)
    return match.group(1).strip() if match else None

def fetch_detail(session, listing: dict) -> dict | None:
    detail_frame = find_frame(session, listing["url"], "Detail")
    resp = session.get(detail_frame, timeout=30)
    if resp.status_code in (401, 404, 410):
        return None
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text, "html.parser")
    blocks = soup.select(".sfdc_richtext, .jobDescription, .jobDescrtionScope")
    blocks = [b for b in blocks if len(b.get_text(strip=True)) >= 80]
    body = max(blocks, key=lambda b: len(b.get_text()), default=None)
    heading = soup.select_one(".divJobDetail h2, h1.job-title, h1")
    if body is None or heading is None:
        return None

    # Salesforce renders adjacent rows without whitespace, so read each row alone.
    texts = [row.get_text() for row in soup.select("table.Job_Detail tr")]
    city = next((v for v in (match_label(t, "City") for t in texts) if v), None)
    state = next((v for v in (match_label(t, "State") for t in texts) if v), None)

    apply_match = APPLY_TARGET.search(resp.text)
    return {
        "id": listing["id"],
        "title": heading.get_text(strip=True),
        "description_html": body.decode_contents().strip(),
        "city": city or listing.get("city"),
        "state": state or listing.get("state"),
        "url": listing["url"],
        "apply_url": (urljoin(listing["url"], apply_match.group(1))
                      if apply_match else listing["url"]),
    }