Career Sites Feed API

"""Incremental sync: pages newly discovered / updated career sites since the last run.
   Cursor is checkpointed so a crash resumes mid-stream."""
import json, time, datetime as dt, requests, pathlib

API_KEY = "YOUR_API_KEY"
URL = "https://connect.jobo.world/api/career-sites/feed"
STATE = pathlib.Path("career_sites_sync_state.json")
OVERLAP = dt.timedelta(minutes=15)

state = json.loads(STATE.read_text()) if STATE.exists() else {"last_run_at": None, "cursor": None}
this_run_at = dt.datetime.now(dt.timezone.utc).isoformat()
store = {}  # ← your DB. key = company["id"]

def post(body):
    r = requests.post(URL,
                      headers={"X-Api-Key": API_KEY, "Content-Type": "application/json"},
                      json=body)
    r.raise_for_status()
    return r.json()

updated_after = None
if state["last_run_at"]:
    updated_after = (dt.datetime.fromisoformat(state["last_run_at"]) - OVERLAP).isoformat()

cursor = state.get("cursor")
while True:
    body = {"batch_size": 50}
    if updated_after: body["updated_after"] = updated_after
    if cursor:        body["cursor"] = cursor
    data = post(body)

    for site in data["career_sites"]:
        store[site["id"]] = site  # ← upsert in your DB

    cursor = data["next_cursor"]
    STATE.write_text(json.dumps({**state, "cursor": cursor}))  # checkpoint
    if not data["has_more"]: break

# Commit the new checkpoint only after the sync succeeded.
STATE.write_text(json.dumps({"last_run_at": this_run_at, "cursor": None}))
print(f"Synced. {len(store)} career sites in store.")