Jobtrain Jobs API.

import requests
from urllib.parse import urlsplit

def board_urls(board_url: str) -> tuple[str, str]:
    # "https://www.jobtrain.co.uk/sigplc/Home/Job"  (path-hosted)
    # "https://hft.jobtrain.co.uk/Home/Job"          (subdomain-hosted)
    parts = urlsplit(board_url)
    origin = f"{parts.scheme}://{parts.netloc}"
    prefix = parts.path[: -len("/Job")]  # ".../Home"
    listing_url = f"{origin}{prefix}/Job"
    card_url = f"{origin}{prefix}/_JobCard"
    return listing_url, card_url

listing_url, card_url = board_urls("https://www.jobtrain.co.uk/sigplc/Home/Job")
print(card_url)
# https://www.jobtrain.co.uk/sigplc/Home/_JobCard

from bs4 import BeautifulSoup

PAGE_SIZE = 12
MAX_PAGES = 200  # 12 * 200 = 2,400-record audited safety cap

def hidden_int(soup, element_id):
    el = soup.select_one(f"#{element_id}")
    value = el.get("value") if el is not None else None
    return int(value) if value and value.isdigit() else None

cards, total, received = [], None, 0
for page in range(MAX_PAGES):
    resp = requests.get(f"{card_url}?Skip={page * PAGE_SIZE}", timeout=30)
    if resp.status_code in (404, 410):
        raise SystemExit("Tenant board not found")
    if resp.status_code in (401, 403, 429):
        raise SystemExit(f"Blocked or auth-gated (HTTP {resp.status_code})")
    resp.raise_for_status()
    if not resp.text.strip():
        break  # empty page terminates pagination

    soup = BeautifulSoup(resp.text, "html.parser")
    page_total = hidden_int(soup, "totalMatchRecords")
    current = hidden_int(soup, "totalCurrentRecords")
    page_cards = soup.select("div.job-card")
    if page_total is None or current is None or current != len(page_cards):
        raise SystemExit("Incomplete page — missing/mismatched count markers")

    total = page_total if total is None else total
    received += len(page_cards)
    cards.extend(page_cards)
    if received >= total or len(page_cards) < PAGE_SIZE:
        break

print(f"Collected {received} of {total} advertised jobs")

import re
from urllib.parse import urljoin

JOB_ID_RE = re.compile(r"[?&]JobId=(\d+)", re.IGNORECASE)

def detail_item(card, class_name):
    el = card.select_one(f".jobdetailsitem.{class_name}")
    if el is None:
        return None
    full = el.get_text(" ", strip=True)
    label_el = el.select_one("strong")
    label = label_el.get_text(" ", strip=True) if label_el is not None else ""
    value = full[len(label):].strip() if label and full.startswith(label) else full
    return value or None

jobs = []
for card in cards:
    anchor = (card.select_one("a[data-testid^='a-job-detail-']")
              or card.select_one("a[href*='JobDetail']"))
    href = anchor.get("href") if anchor is not None else None
    if not href:
        continue
    detail_url = urljoin(listing_url, href)
    match = JOB_ID_RE.search(detail_url)
    title = anchor.get_text(" ", strip=True) if anchor is not None else None
    if not (match and title):
        continue  # skip cards with no numeric JobId or title
    loc = card.select_one(".job-card__location strong")
    jobs.append({
        "external_id": match.group(1),
        "title": title,
        "location": loc.get_text(" ", strip=True) if loc is not None else None,
        "reference": detail_item(card, "jobreference"),
        "employment_type": detail_item(card, "employmenttype"),
        "detail_url": detail_url,
    })

print(f"Parsed {len(jobs)} cards")

import json

def scrape_detail(job: dict) -> dict:
    resp = requests.get(job["detail_url"], timeout=30)
    if resp.status_code in (404, 410) or not resp.text.strip():
        return {**job, "removed": True}
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text, "html.parser")
    posting = None
    for tag in soup.select('script[type="application/ld+json"]'):
        try:
            data = json.loads(tag.string or "")
        except (ValueError, TypeError):
            continue
        for node in (data if isinstance(data, list) else [data]):
            if isinstance(node, dict) and node.get("@type") == "JobPosting":
                posting = node
                break
        if posting:
            break
    if posting is None:
        raise ValueError("No JSON-LD JobPosting on detail page")

    org = posting.get("hiringOrganization") or {}
    raw_locs = posting.get("jobLocation")
    raw_locs = raw_locs if isinstance(raw_locs, list) else [raw_locs] if raw_locs else []

    apply_anchor = (
        soup.select_one("a[data-testid='a-btn-decide-internal-external'][href*='JobId=']")
        or soup.select_one("a[href*='/DecideInternalExternal/'][href*='JobId=']")
    )
    apply_url = urljoin(job["detail_url"], apply_anchor["href"]) if apply_anchor else None

    return {
        **job,
        "description_html": posting.get("description"),
        "posted_at": posting.get("datePosted"),
        "closes_at": posting.get("validThrough"),
        "employment_type": posting.get("employmentType") or job.get("employment_type"),
        "salary": posting.get("baseSalary"),   # free-text string, not a schema.org object
        "company_name": org.get("name"),
        "logo": org.get("logo"),
        "locations": [loc.get("address", {}) for loc in raw_locs if isinstance(loc, dict)],
        "apply_url": apply_url,
    }

detailed = [scrape_detail(j) for j in jobs]
print(f"Scraped {len(detailed)} Jobtrain job details")