Feed API

"""Incremental sync: pages new/updated jobs since the last run, then sweeps expired IDs.
   Cursor is checkpointed so a crash resumes mid-stream."""
import json, time, datetime as dt, requests, pathlib

API_KEY = "YOUR_API_KEY"
BASE = "https://connect.jobo.world/api/jobs"
STATE = pathlib.Path("sync_state.json")
OVERLAP = dt.timedelta(minutes=15)

state = json.loads(STATE.read_text()) if STATE.exists() else {"last_run_at": None, "cursor": None}
this_run_at = dt.datetime.now(dt.timezone.utc).isoformat()
store = {}  # ← your DB. key = job["id"]

def post(body):
    while True:
        r = requests.post(f"{BASE}/feed",
                          headers={"X-Api-Key": API_KEY, "Content-Type": "application/json"},
                          json=body)
        if r.status_code == 503:
            time.sleep(int(r.headers.get("Retry-After", "5"))); continue
        r.raise_for_status(); return r.json()

# 1. Page the feed with posted_after = last_run - overlap
posted_after = None
if state["last_run_at"]:
    posted_after = (dt.datetime.fromisoformat(state["last_run_at"]) - OVERLAP).isoformat()

cursor = state.get("cursor")
while True:
    body = {"batch_size": 1000}
    if posted_after: body["posted_after"] = posted_after
    if cursor:       body["cursor"] = cursor
    data = post(body)

    for job in data["jobs"]:
        store[job["id"]] = job  # ← upsert in your DB

    cursor = data["next_cursor"]
    STATE.write_text(json.dumps({**state, "cursor": cursor}))  # checkpoint
    if not data["has_more"]: break

# 2. Sweep expired IDs since the last successful run
if state["last_run_at"]:
    cursor = None
    while True:
        params = {"expired_since": state["last_run_at"], "batch_size": 10000}
        if cursor: params["cursor"] = cursor
        r = requests.get(f"{BASE}/expired",
                         headers={"X-Api-Key": API_KEY}, params=params).json()
        for jid in r["job_ids"]:
            store.pop(jid, None)  # ← mark expired in your DB
        cursor = r["next_cursor"]
        if not r["has_more"]: break

# 3. Commit the new checkpoint only after both halves succeeded.
STATE.write_text(json.dumps({"last_run_at": this_run_at, "cursor": None}))
print(f"Synced. {len(store)} active jobs in store.")