Surface stale deadlines instead of showing them as upcoming

The calendar's deadline table rendered every recorded submission cycle, so
elapsed rounds sorted to the top and read as the next thing due — 13 of its 17
rows were already in the past. Underneath that, the fetcher preserves
`source: manual` entries forever without ever checking whether they still
describe a future deadline, and 11 of 13 cloud-edge venues are manual.

Preserving them is correct: WikiCFP cannot express multi-cycle venues and does
not carry most systems conferences at all, so re-fetching would replace
researched rounds with worse data or nothing. What was missing is a signal.

- Extract the cycle helpers into cycles.py so the fetcher and the generator
  share one definition of which round a reader should act on. cycles_of()
  reads an entry without folding flat fields into a cycles list, which would
  otherwise rewrite every single-round entry the fetcher writes back.
- The calendar table lists open cycles only; a venue whose every round has
  elapsed moves to an "Awaiting the next call" table instead of vanishing.
- pa-fetch-deadlines records those venues under a new stale: key and probes
  the year-bumped CFP URL to say which ones already have a next edition
  online. A candidate must return 200, mention the target year, and be newer
  than any year the entry records — venues.yaml still points SC at sc24.
- README: Task 7 and a matching agent prompt for keeping deadlines current,
  since nothing in this repo refreshes them on its own.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/publish_assistant/fetch_deadlines.py

@@ -10,8 +10,15 @@ For each conference in data/venues.yaml, this script:
 Conferences where no CFP was found are listed under the 'missing' key in the
 output YAML — the agent should fill those in manually (Task 2 in README).
 
+Entries whose every recorded round has already elapsed are listed under 'stale'.
+Manual entries are preserved untouched (WikiCFP cannot express multi-cycle
+venues, so re-fetching them would destroy hand-researched rounds) — reporting is
+all this script can do for them. Where the venue's CFP URL carries a year, the
+next edition's URL is probed so the report says which ones are worth researching
+now.
+
 Usage:
-    python fetch_deadlines.py [--venues PATH] [--output PATH]
+    python fetch_deadlines.py [--venues PATH] [--output PATH] [--no-probe]
 """
 import argparse
 import re
@@ -24,6 +31,8 @@ import requests
 import yaml
 from bs4 import BeautifulSoup
 
+from publish_assistant.cycles import cycle_deadline, cycles_of, next_cycle, open_cycles
+
 WIKICFP_SEARCH = "http://www.wikicfp.com/cfp/servlet/tool.search"
 WIKICFP_EVENT = "http://www.wikicfp.com/cfp/servlet/event.showcfp"
 WIKICFP_SERIES = "http://www.wikicfp.com/cfp/program?id={series_id}"
@@ -127,6 +136,87 @@ def best_match(results: list[dict], acronym: str) -> dict | None:
     return None
 
 
+# A four-digit year anywhere in the URL ('/2026/', '2027.eurosys.org'), or a
+# two-digit one glued to the end of a name ('sc26.supercomputing.org', 'osdi27').
+_YEAR_RE = re.compile(r"(?<!\d)(20\d{2})(?!\d)")
+_SHORT_YEAR_RE = re.compile(r"(?<=[a-z])(\d{2})(?!\d)", re.I)
+
+
+def next_edition_urls(url: str) -> list[tuple[str, int]]:
+    """Candidate URLs for the edition after the one `url` describes.
+
+    Venue sites are year-stamped, so the next edition usually lives at the same
+    URL with the year advanced. Returns (url, year) pairs; empty when the URL
+    carries no year to bump — HPDC, for instance, moves to a new host each year.
+    """
+    candidates: list[tuple[str, int]] = []
+
+    years = [int(y) for y in _YEAR_RE.findall(url)]
+    if years:
+        # Bump every occurrence: IPDPS carries the year twice in one URL.
+        bumped = _YEAR_RE.sub(lambda m: str(int(m.group(1)) + 1), url)
+        candidates.append((bumped, max(years) + 1))
+
+    short = _SHORT_YEAR_RE.search(url)
+    if short:
+        bumped = _SHORT_YEAR_RE.sub(lambda m: f"{int(m.group(1)) + 1:02d}", url)
+        candidates.append((bumped, 2000 + int(short.group(1)) + 1))
+
+    return [(u, y) for u, y in candidates if u != url]
+
+
+def probe(session: requests.Session, url: str, year: int) -> bool:
+    """True when `url` serves a real page for `year`.
+
+    GET rather than HEAD — some venue hosts answer HEAD with 405. The year has
+    to appear in the body too: a host that serves its current landing page for
+    any path would otherwise read as next year's site being live.
+    """
+    try:
+        r = session.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
+    except requests.RequestException:
+        return False
+    return r.status_code == 200 and str(year) in r.text
+
+
+def recorded_year(entry: dict) -> int:
+    """The latest year this entry already describes.
+
+    A probe result only counts if it is newer than this. Venue URLs sometimes
+    lag several editions behind — `venues.yaml` still points SC at sc24 — and
+    bumping one of those by a year lands on a conference that already happened.
+    """
+    dates = [entry.get("event_dates") or ""]
+    dates += [cycle_deadline(c) for c in cycles_of(entry)]
+    years = [int(y) for y in _YEAR_RE.findall(" ".join(dates))]
+    return max(years, default=0)
+
+
+def find_next_edition(session: requests.Session, entry: dict, conf: dict) -> str | None:
+    """URL of the next edition's call, if it is already online."""
+    floor = recorded_year(entry)
+    for source in (entry.get("cfp_url"), conf.get("url")):
+        for url, year in next_edition_urls(source or ""):
+            if year > floor and probe(session, url, year):
+                return url
+    return None
+
+
+def stale_record(session: requests.Session, acronym: str, entry: dict, conf: dict,
+                 probe_enabled: bool) -> dict:
+    """Describe an entry whose every recorded round has elapsed."""
+    record = {"acronym": acronym,
+              "last_deadline": cycle_deadline(next_cycle(entry)) or "TBD",
+              "source": entry.get("source") or "unknown"}
+    if entry.get("cfp_url"):
+        record["cfp_url"] = entry["cfp_url"]
+    if probe_enabled:
+        nxt = find_next_edition(session, entry, conf)
+        if nxt:
+            record["next_edition_url"] = nxt
+    return record
+
+
 def process_venue(session: requests.Session, conf: dict) -> dict | None:
     """Return deadline data for a conference, or None if not found."""
     acronym = conf.get("acronym", "")
@@ -157,6 +247,8 @@ def main() -> None:
     ap = argparse.ArgumentParser(description="Fetch submission deadlines from WikiCFP.")
     ap.add_argument("--venues", default="site/data/cloud-edge/venues.yaml")
     ap.add_argument("--output", default="site/data/cloud-edge/deadlines.yaml")
+    ap.add_argument("--no-probe", action="store_true",
+                    help="skip the HTTP check for a stale venue's next-edition page")
     args = ap.parse_args()
 
     venues_path = Path(args.venues)
@@ -185,12 +277,23 @@ def main() -> None:
     session = requests.Session()
     found: dict[str, dict] = dict(existing)  # start with manual entries
     missing: list[str] = []
+    stale: list[dict] = []
+    probe_enabled = not args.no_probe
 
     for conf in conferences:
         acronym = conf.get("acronym", "")
         print(f"  {acronym} …", file=sys.stderr)
         if acronym in existing:
-            print(f"    manual entry preserved", file=sys.stderr)
+            # Preserved verbatim either way — all this run can do is say whether
+            # the entry still describes a deadline anyone can act on.
+            entry = existing[acronym]
+            if open_cycles(entry):
+                print("    manual entry preserved", file=sys.stderr)
+            else:
+                record = stale_record(session, acronym, entry, conf, probe_enabled)
+                stale.append(record)
+                print(f"    manual entry preserved — STALE since {record['last_deadline']}",
+                      file=sys.stderr)
             continue
         try:
             time.sleep(REQUEST_DELAY)
@@ -199,6 +302,11 @@ def main() -> None:
                 found[acronym] = data
                 dl = data.get("submission_deadline") or data.get("abstract_deadline") or "?"
                 print(f"    deadline: {dl}", file=sys.stderr)
+                # WikiCFP IDs are edition-specific, so a pinned ID keeps serving
+                # a conference that already happened. Same symptom, same report.
+                if not open_cycles(data):
+                    stale.append(stale_record(session, acronym, data, conf, probe_enabled))
+                    print(f"    STALE — this edition has already run", file=sys.stderr)
             else:
                 missing.append(acronym)
                 print(f"    not found on WikiCFP", file=sys.stderr)
@@ -206,10 +314,14 @@ def main() -> None:
             missing.append(acronym)
             print(f"    error: {exc}", file=sys.stderr)
 
+    # Actionable first: venues whose next edition is already online.
+    stale.sort(key=lambda r: ("next_edition_url" not in r, r["acronym"]))
+
     output = {
         "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
         "deadlines": found,
         "missing": missing,
+        "stale": stale,
     }
 
     out_path = Path(args.output)
@@ -218,7 +330,8 @@ def main() -> None:
         yaml.dump(output, f, allow_unicode=True, sort_keys=False, default_flow_style=False)
 
     print(
-        f"\nResults: {len(found)} found, {len(missing)} missing → {out_path}",
+        f"\nResults: {len(found)} found, {len(missing)} missing, {len(stale)} stale "
+        f"→ {out_path}",
         file=sys.stderr,
     )
     if missing:
@@ -226,6 +339,18 @@ def main() -> None:
             f"Missing (fill in manually — see README Task 2): {', '.join(missing)}",
             file=sys.stderr,
         )
+    if stale:
+        print("\nStale — every recorded round has elapsed (see README Task 7):",
+              file=sys.stderr)
+        for record in stale:
+            if record.get("next_edition_url"):
+                note = f"NEW EDITION: {record['next_edition_url']}"
+            elif probe_enabled:
+                note = "next edition not announced yet"
+            else:
+                note = "next edition not checked (--no-probe)"
+            print(f"  {record['acronym']:<12} last deadline "
+                  f"{record['last_deadline']:<16} {note}", file=sys.stderr)
 
 
 if __name__ == "__main__":