SAP SuccessFactors Jobs API.

import requests
import re

# Global:  https://{tenant}.jobs.hr.cloud.sap
# China:   https://{tenant}.jobs.hr.sapcloud.cn
base_url = "https://wlgore.jobs.hr.cloud.sap"

session = requests.Session()
session.headers.update({"User-Agent": "Mozilla/5.0 (compatible; JoboBot/1.0)"})

# The homepage GET sets JSESSIONID on the cookie jar and embeds the CSRF token.
resp = session.get(base_url)
resp.raise_for_status()
html = resp.text

# The token lives in one of three markers; try them in order.
csrf_token = None
for pattern in (
    r'<meta\s+name="csrf-token"\s+content="([^"]+)"',
    r'data-csrf-token="([^"]+)"',
    r'x-csrf-token:\s*([a-f0-9-]+)',
):
    m = re.search(pattern, html, re.IGNORECASE)
    if m:
        csrf_token = m.group(1)
        break

print("CSRF token:", csrf_token)

# Prefer /go/{name}/{id} nav links, then fall back to embedded categoryId JSON.
category_ids = set()
for m in re.finditer(r'/go/[^/"]+/(\d+)/?', html):
    category_ids.add(int(m.group(1)))

if not category_ids:
    for m in re.finditer(r'"categoryId"\s*:\s*(\d+)', html):
        category_ids.add(int(m.group(1)))

# categoryId 0 is the "all jobs" fallback when no categories are advertised.
category_ids = sorted(category_ids) or [0]
print("Categories:", category_ids)

def fetch_page(session, base_url, csrf_token, category_id, page_number):
    api_url = f"{base_url.rstrip('/')}/services/recruiting/v1/jobs"
    headers = {"Content-Type": "application/json"}
    if csrf_token:
        headers["x-csrf-token"] = csrf_token
    payload = {
        "locale": "en_US",
        "pageNumber": page_number,
        "sortBy": "",
        "keywords": "",
        "location": "",
        "facetFilters": {},
        "brand": "",
        "skills": [],
        "categoryId": category_id,
        "alertId": "",
        "rcmCandidateId": "",
    }
    # JSESSIONID rides along automatically from the session cookie jar.
    r = session.post(api_url, headers=headers, json=payload)
    r.raise_for_status()
    return r.json()

data = fetch_page(session, base_url, csrf_token, category_ids[0], 0)
results = data.get("jobSearchResult", [])
total_jobs = data.get("totalJobs", 0)
print(f"{len(results)} jobs on page 0 of {total_jobs} total")

def clean_location(job):
    short = job.get("jobLocationShort") or []
    if short and short[0]:
        return re.sub(r"<br\s*/?>", "", short[0]).strip()
    state = (job.get("jobLocationState") or [None])[0]
    country = (job.get("jobLocationCountry") or [None])[0]
    return ", ".join(p for p in (state, country) if p) or None

parsed = []
for item in results:
    job = item.get("response") or {}
    if not job.get("id"):
        continue  # drop rows without a stable native id
    parsed.append({
        "id": job["id"],
        "title": (job.get("unifiedStandardTitle") or job.get("unifiedUrlTitle") or "").strip(),
        "location": clean_location(job),
        "business_unit": ", ".join(job.get("businessUnit_obj") or []),
        "department": ", ".join(job.get("filter2") or []),
        "subcategory": ", ".join(job.get("filter8") or []),
        "currency": ", ".join(job.get("currency") or []),
        "posted_date": job.get("unifiedStandardStart"),  # US M/d/yy
        "locales": job.get("supportedLocales") or [],
        "url_title": job.get("unifiedUrlTitle") or job.get("urlTitle") or "job",
    })

print(f"Parsed {len(parsed)} listings")

from bs4 import BeautifulSoup

def fetch_details(session, base_url, job):
    locale = job["locales"][0] if job["locales"] else "en_US"
    detail_url = f"{base_url.rstrip('/')}/job/{job['url_title']}/{job['id']}-{locale}"
    r = session.get(detail_url)
    r.raise_for_status()
    soup = BeautifulSoup(r.text, "html.parser")

    h1 = soup.find("h1")
    title = h1.get_text(strip=True) if h1 else job["title"]

    # Descriptions live in .job-description, or the broader .job container.
    desc_el = soup.select_one("div.job-description") or soup.select_one("div.job")
    description = desc_el.get_text(" ", strip=True) if desc_el else None

    # Labelled <h2> sections expose structured fields.
    fields = {}
    for h2 in soup.find_all("h2"):
        label = h2.get_text(strip=True).rstrip(":").lower()
        value = " ".join(
            sib.get_text(" ", strip=True)
            for sib in h2.find_next_siblings()
            if sib.name != "h2"
        ).strip()
        if label in ("req id", "requisition id", "job id"):
            fields["req_id"] = value
        elif label in ("work arrangement", "workplace type", "work type"):
            fields["work_arrangement"] = value
        elif label in ("posting start date", "posted date", "date posted"):
            fields["posted_date"] = value

    return {**job, "title": title, "description": description,
            "detail_url": detail_url, **fields}

detailed = fetch_details(session, base_url, parsed[0])
print(f"Description: {len(detailed.get('description') or '')} chars")

import time

JOBS_PER_PAGE = 50

def fetch_all(session, base_url, csrf_token, category_ids):
    all_jobs = []
    for category_id in category_ids:
        page = 0
        while True:
            data = fetch_page(session, base_url, csrf_token, category_id, page)
            results = data.get("jobSearchResult", [])
            if not results:
                break  # defensive: stop this category on an empty page
            all_jobs.extend(results)
            total = data.get("totalJobs", 0)
            fetched = page * JOBS_PER_PAGE + len(results)
            print(f"cat {category_id} page {page}: +{len(results)} ({fetched}/{total})")
            if total and fetched >= total:
                break
            page += 1
            time.sleep(0.2)  # match the scraper's 200ms inter-request delay
    return all_jobs

jobs = fetch_all(session, base_url, csrf_token, category_ids)
print(f"Total jobs fetched: {len(jobs)}")