Recooty Jobs API.

import requests
import xml.etree.ElementTree as ET
from urllib.parse import urlparse

def list_company_jobs(host, company_slug):
    """Enumerate a company's job URLs from the per-host sitemap.

    Recooty publishes one sitemap per host listing every job on the domain.
    Keep only <loc> entries whose first path segment is this company and that
    point at a job detail page: exactly /{company}/{job-slug}, never /apply."""
    sitemap_url = f"https://{host}/sitemap.xml"
    resp = requests.get(sitemap_url, timeout=30)
    resp.raise_for_status()

    root = ET.fromstring(resp.content)
    listings = []
    seen = set()

    for loc in root.iter():
        # Sitemaps may or may not declare a namespace; match on the local name.
        if loc.tag.split("}")[-1] != "loc":
            continue
        url = (loc.text or "").strip()
        parsed = urlparse(url)
        if parsed.netloc.lower() != host.lower():
            continue

        segments = [s for s in parsed.path.split("/") if s]
        # Job detail pages are exactly /{company}/{job-slug}. Skip the board
        # root (1 segment), deeper paths, and /apply URLs.
        if len(segments) != 2 or segments[0].lower() != company_slug.lower():
            continue
        if segments[1].lower() == "apply":
            continue

        external_id = segments[1]  # e.g. frontend-developer-reactjs-re144
        if external_id in seen:
            continue
        seen.add(external_id)

        canonical = f"https://{host}/{segments[0]}/{segments[1]}"
        listings.append({
            "external_id": external_id,
            "listing_url": canonical,
            "apply_url": canonical + "/apply/",
        })

    return listings

jobs = list_company_jobs("careerspage.io", "recooty-inc")
print(f"Found {len(jobs)} jobs")

import requests

def fetch_detail_html(listing_url, board_url):
    """Fetch a job detail page. The canonical listing URL IS the detail page;
    Recooty has no structured detail JSON endpoint."""
    resp = requests.get(
        listing_url,
        headers={
            "Accept": "text/html,application/xhtml+xml",
            "Referer": board_url,
        },
        timeout=30,
    )
    resp.raise_for_status()
    return resp.text

html = fetch_detail_html(jobs[0]["listing_url"], "https://careerspage.io/recooty-inc")
print(f"Fetched {len(html)} bytes")

from bs4 import BeautifulSoup

def parse_detail_html(html):
    soup = BeautifulSoup(html, "html.parser")

    def text_of(*selectors):
        for sel in selectors:
            el = soup.select_one(sel)
            if el:
                t = el.get_text(strip=True)
                if t:
                    return t
        return None

    # Title and location fall back from the scoped selector to the bare one.
    title = text_of("div.pt-9 h3.font-size-6", "h3.font-size-6")
    location = text_of("div.pt-9 span.font-size-3.text-gray",
                       "span.font-size-3.text-gray")

    # Description is the inner HTML of company-profile (fallback job-details-content).
    body = (soup.select_one("div.company-profile")
            or soup.select_one("div.job-details-content"))
    description_html = body.decode_contents().strip() if body else ""

    # Sidebar metadata is keyed off FontAwesome icons, not text labels.
    def meta_for(*icon_classes):
        for icon in icon_classes:
            i = soup.select_one(f"i.{icon}")
            if not i:
                continue
            container = i.find_parent(["p", "div", "li"])
            if container:
                val = container.get_text(" ", strip=True)
                if val:
                    return val
        return None

    employment_type = meta_for("fa-briefcase")
    experience_level = meta_for("fa-business-time", "fa-user-tie")
    location_type = meta_for("fa-location-arrow")

    apply_el = (soup.select_one('a.btn-green[href*="/apply"]')
                or soup.select_one('a.btn[href*="/apply"]'))
    apply_url = apply_el["href"] if apply_el and apply_el.has_attr("href") else None

    return {
        "title": title,
        "location": location,
        "description_html": description_html,
        "employment_type": employment_type,
        "experience_level": experience_level,
        "location_type": location_type,
        "apply_url": apply_url,
    }

detail = parse_detail_html(html)
print(detail["title"], detail["location"])

import re

# Recooty's application/ld+json is frequently INVALID json (raw newlines and
# unescaped HTML inside "description"), so json.loads() raises. Pull the
# well-formed top-level string fields with regex instead of parsing the object.
LD_BLOCK = re.compile(r'<script[^>]*application/ld\+json[^>]*>(.*?)</script>',
                      re.IGNORECASE | re.DOTALL)

def _ld_string(body, key):
    m = re.search(r'"' + re.escape(key) + r'"\s*:\s*"((?:\\.|[^"\\])*)"', body)
    return m.group(1) if m else None

def _ld_object(body, key):
    # Scope an inner object (assumes no nested braces, true for Recooty blocks).
    m = re.search(r'"' + re.escape(key) + r'"\s*:\s*\{([^{}]*)\}', body)
    return m.group(1) if m else None

def extract_jsonld_fields(html):
    for block in LD_BLOCK.finditer(html):
        body = block.group(1)
        if "JobPosting" not in body:
            continue
        # hiringOrganization.name is scoped so it never collides with a
        # Place/PostalAddress "name" elsewhere in the block.
        org = _ld_object(body, "hiringOrganization")
        return {
            "company_name": _ld_string(org, "name") if org else None,
            "date_posted": _ld_string(body, "datePosted"),
            "valid_through": _ld_string(body, "validThrough"),
            "employment_type": _ld_string(body, "employmentType"),
            "experience_requirements": _ld_string(body, "experienceRequirements"),
            "job_location_type": _ld_string(body, "jobLocationType"),
            "address_locality": _ld_string(body, "addressLocality"),
            "address_region": _ld_string(body, "addressRegion"),
            "address_country": _ld_string(body, "addressCountry"),
        }
    return {}

ld = extract_jsonld_fields(html)
print(ld.get("date_posted"), ld.get("valid_through"))

import time
import requests

def get_with_backoff(url, headers=None, max_attempts=3):
    """On HTTP 429, honor Retry-After, falling back to 2s then 5s (clamped)."""
    fallback = [2.0, 5.0]
    resp = None
    for attempt in range(max_attempts):
        resp = requests.get(url, headers=headers, timeout=30)
        if resp.status_code != 429 or attempt == max_attempts - 1:
            return resp
        retry_after = resp.headers.get("Retry-After")
        try:
            delay = float(retry_after) if retry_after else fallback[min(attempt, 1)]
        except ValueError:
            delay = fallback[min(attempt, 1)]
        time.sleep(max(0.5, min(delay, 30.0)))
    return resp

def scrape_company(host, company_slug):
    board_url = f"https://{host}/{company_slug}"
    results = []
    for job in list_company_jobs(host, company_slug):
        resp = get_with_backoff(
            job["listing_url"],
            headers={"Accept": "text/html,application/xhtml+xml", "Referer": board_url},
        )
        if not resp.ok:
            continue  # 404 -> removed; 403/429 -> throttled
        detail = parse_detail_html(resp.text)
        detail.update(extract_jsonld_fields(resp.text))
        detail["external_id"] = job["external_id"]
        detail["listing_url"] = job["listing_url"]
        detail["apply_url"] = detail.get("apply_url") or job["apply_url"]
        results.append(detail)
        time.sleep(0.75)  # 1 request at a time, 750ms apart
    return results

all_jobs = scrape_company("careerspage.io", "recooty-inc")
print(f"Scraped {len(all_jobs)} jobs")