From 5b06f484f5639c84991f86c43e0a0dbcd2f451f4 Mon Sep 17 00:00:00 2001 From: Vincent Lannurien Date: Mon, 17 Aug 2026 22:25:24 +0200 Subject: [PATCH] Surface stale deadlines instead of showing them as upcoming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The calendar's deadline table rendered every recorded submission cycle, so elapsed rounds sorted to the top and read as the next thing due — 13 of its 17 rows were already in the past. Underneath that, the fetcher preserves `source: manual` entries forever without ever checking whether they still describe a future deadline, and 11 of 13 cloud-edge venues are manual. Preserving them is correct: WikiCFP cannot express multi-cycle venues and does not carry most systems conferences at all, so re-fetching would replace researched rounds with worse data or nothing. What was missing is a signal. - Extract the cycle helpers into cycles.py so the fetcher and the generator share one definition of which round a reader should act on. cycles_of() reads an entry without folding flat fields into a cycles list, which would otherwise rewrite every single-round entry the fetcher writes back. - The calendar table lists open cycles only; a venue whose every round has elapsed moves to an "Awaiting the next call" table instead of vanishing. - pa-fetch-deadlines records those venues under a new stale: key and probes the year-bumped CFP URL to say which ones already have a next edition online. A candidate must return 200, mention the target year, and be newer than any year the entry records — venues.yaml still points SC at sc24. - README: Task 7 and a matching agent prompt for keeping deadlines current, since nothing in this repo refreshes them on its own. Co-Authored-By: Claude Opus 5 --- README.md | 82 ++++++++++- site/content/cloud-edge/calendar/_index.md | 31 +++-- site/data/cloud-edge/deadlines.yaml | 57 ++++++-- src/publish_assistant/cycles.py | 101 ++++++++++++++ src/publish_assistant/fetch_deadlines.py | 131 +++++++++++++++++- src/publish_assistant/generate_content.py | 151 +++++++++------------ 6 files changed, 431 insertions(+), 122 deletions(-) create mode 100644 src/publish_assistant/cycles.py diff --git a/README.md b/README.md index 78a01af..d5fce29 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,9 @@ A venue whose `deadlines.yaml` entry says `source: manual` is skipped by the fet **Conferences not on WikiCFP** (for systems/networking): OSDI, NSDI, USENIX ATC, SC, MobiSys, SEC. Use `wikicfp_id: false` for all of them. -**Manual deadline entries**: Add entries with `source: manual` to `site/data//deadlines.yaml`. The fetcher preserves all `source: manual` entries across runs. Format: +**Manual deadline entries**: Add entries with `source: manual` to `site/data//deadlines.yaml`. The fetcher preserves all `source: manual` entries across runs — indefinitely, and by design: WikiCFP cannot express multi-cycle venues and does not carry most systems conferences at all, so re-fetching a manual entry would replace researched data with worse data or nothing. Nothing refreshes these but a person or an agent. + +Because of that, a manual entry can sit unchanged long after its deadline passes. The fetcher does not fix this, but it does **report** it: any entry whose recorded rounds have all elapsed is listed under `stale:` in `deadlines.yaml` and printed at the end of the run. See Task 7. Format: ```yaml ATC: source: manual @@ -202,7 +204,7 @@ What the generator does with cycles: | --- | --- | | Venue page | Multi-cycle venues get a **Submission Cycles** table with every round; the next open one is bolded and marked `(next)`. Single-cycle venues keep the **Upcoming Deadline** block unchanged. | | Venue index | One row per venue, showing the next open cycle with its name appended to the paper deadline. | -| Calendar table | One row per cycle, with a `Cycle` column, sorted by paper deadline. | +| Calendar table | One row per **still-open** cycle, with a `Cycle` column, sorted by paper deadline. A venue whose every round has elapsed drops out of this table into an **Awaiting the next call** table below it, showing the last round that ran — an elapsed round sorts to the top by date and would otherwise read as the next thing due. | | Calendar grid | One event set per cycle; titles read `EuroSys Fall — paper deadline`. Elapsed cycles are kept — the grid is time-indexed, so past rounds only show when you navigate back to their month. | | Front matter | `next_deadline` / `abstract_deadline` / `notification` / `camera_ready` reflect the **next** cycle. Multi-cycle venues additionally carry `cycles` and `next_cycle`. | @@ -259,7 +261,7 @@ Every field must come from a published CFP. There is no fetcher for these, so no Special issue deadlines appear in three places: - **Journal page** — a "Special Issues" table below the journal metadata -- **Calendar page** — a "Journal Special Issues" table below the conference deadlines. The heading always renders; with no entries recorded it carries a line saying no call is open, so the page doesn't read as if journals went untracked +- **Calendar page** — a "Journal Special Issues" table below the conference deadlines (which themselves split into open cycles and "Awaiting the next call"). The heading always renders; with no entries recorded it carries a line saying no call is open, so the page doesn't read as if journals went untracked - **FullCalendar grid** — events colored purple (`#7b2cbf`), titled `ACR SI — Title — deadline type`, carrying `cfp_url` in `extendedProps` Dates use the same `MMM D, YYYY` format as conference deadlines. `notification` additionally accepts month-only precision (`Jan 2027`), which lands on the first of that month in the calendar grid; the tables always print the raw string. @@ -308,9 +310,22 @@ Fetches submission deadlines from WikiCFP for a specific topic. Preserves `sourc ```bash uv run pa-fetch-deadlines \ --venues site/data/cloud-edge/venues.yaml \ - --output site/data/cloud-edge/deadlines.yaml + --output site/data/cloud-edge/deadlines.yaml \ + [--no-probe] ``` -After running: check `deadlines.yaml` for the `missing:` list, then do Task 2. + +Writes three report keys alongside `deadlines:`: + +| Key | Meaning | +| --- | --- | +| `missing:` | No CFP found at all — acronyms only. Fill in by hand (Task 2). | +| `stale:` | The entry exists but every recorded round has elapsed. One map per venue: `acronym`, `last_deadline`, `source`, `cfp_url`, and `next_edition_url` when the probe found one. Refresh by hand (Task 7). | + +**Next-edition probe**: for stale venues only, the script bumps the year in the recorded `cfp_url` (and, failing that, the venue's `url`) and checks whether that page exists — `mobisys/2026/` → `mobisys/2027/`, `sc26.supercomputing.org` → `sc27.…`, `ipdps2027/2027-call-for-papers.html` → `ipdps2028/2028-…`. A candidate counts only if it returns 200, mentions the target year (guarding against hosts that serve a landing page for any path), and is newer than any year the entry already records (`venues.yaml` URLs lag — SC's still points at sc24). This does not read deadlines; it only tells you which venues are worth researching *now*. `--no-probe` skips it. + +The run ends with a summary naming every missing and stale venue. It always exits 0 — `build.sh` runs under `set -euo pipefail`, so a non-zero exit would abort the build before content is generated. + +After running: check the summary, then do Task 2 (missing) and Task 7 (stale). ### `pa-fetch-best-papers` Scrapes [jeffhuang.com/best_paper_awards/](https://jeffhuang.com/best_paper_awards/) and writes `best_papers.yaml`. Run once per year. @@ -579,6 +594,44 @@ Checklist: --- +### Keep deadlines current + +``` +Refresh the deadlines for the publish-assistant topic "". + +1. Run: ./build.sh --skip-rankings --topic + Read the summary the deadline fetch prints at the end — it lists every venue + under missing: (no entry at all) and stale: (entry frozen on an edition that + already ran), and for stale ones whether the next edition's page is online. + +2. For each missing: venue, visit the official CFP page and add a source: manual + entry to site/data//deadlines.yaml (README Task 2). + +3. For each stale: venue: + - If the report printed "NEW EDITION: ", open it and record the new + round(s): abstract deadline, paper deadline, notification, camera-ready, + event dates, location, cfp_url. + - If it printed "next edition not announced yet", still check the venue site + once — the probe only tries a year-bumped URL, so it misses venues that + move host between editions (HPDC) or use an unrelated page name. + - If the new call genuinely is not out, leave the entry untouched. It shows + under "Awaiting the next call" on the calendar, which is correct. + - Multi-round venues keep source: manual and a cycles: list. Do not hand them + back to WikiCFP — it records only one round and would drop the others. + - Update the venue's url in venues.yaml if it still points at the old edition. + +4. Re-run: ./build.sh --skip-rankings --topic + +Checklist: +- [ ] Every stale: venue either updated or confirmed to have no published CFP +- [ ] missing: list empty, or each entry explained +- [ ] No source: manual entry lost its cycles +- [ ] site/data// and the regenerated site/content// both committed + (CI only runs hugo — it never regenerates content) +``` + +--- + ## Agent Tasks (Reference) ### Task 1 — Bootstrap a topic @@ -586,7 +639,7 @@ See the "Bootstrap a new topic" prompt above. The one-shot checklist is embedded ### Task 2 — Fill in missing deadlines -**Trigger:** `pa-fetch-deadlines` lists conferences under `missing:`, or a deadline looks wrong. +**Trigger:** `pa-fetch-deadlines` lists conferences under `missing:`, or a deadline looks wrong. For venues that *have* an entry whose rounds have all elapsed, see Task 7 instead — same research, different starting point. 1. For each missing conference, visit the official website's "Call for Papers" / "Important Dates" page. 2. Also check WikiCFP manually — if you find the right event ID, add `wikicfp_id` to `venues.yaml` so future runs fetch it automatically. @@ -631,6 +684,21 @@ See the "Add a venue to an existing topic" prompt above. ### Task 6 — Annual cycle refresh See the "Annual cycle refresh for a topic" prompt above. +### Task 7 — Keep deadlines current + +**Trigger:** routine maintenance. Nothing in this repo refreshes deadlines on its own — no cron, no CI job, and manual entries are preserved forever by design (see "Manual deadline entries"). An agent has to run this, and should run it whenever picking the repo up. + +1. Run `./build.sh --skip-rankings` (add `--topic ` to scope it). Read the summary the deadline fetch prints at the end. +2. **`missing:`** — no entry at all. Do Task 2. +3. **`stale:`** — the entry is frozen on an edition that already ran. For each one: + - If the report says `NEW EDITION: `, open it: the next call is already published, so the dates are there to be recorded. + - Otherwise check the venue's site anyway before moving on — the probe only tries a year-bumped URL, so it misses venues that change host between editions (HPDC) or announce on a page with an unrelated name. + - When the new CFP exists, replace that venue's rounds in `deadlines.yaml`, update `event_dates`, `location`, `cfp_url`, and the venue's `url` in `venues.yaml` if it points at the old edition. Keep `source: manual` on multi-cycle venues — see "WikiCFP cannot express this". + - When it does not exist yet, leave the entry alone. It stays under "Awaiting the next call" on the calendar, which is the honest state. +4. Re-run `./build.sh --skip-rankings`, then commit **both** `site/data//` and the regenerated `site/content//` — CI only runs `hugo` and will not regenerate anything. + +Worth re-running even when no deadline changed: the calendar's open/elapsed split is computed against the date of the run, so the committed page drifts as rounds pass. + --- ## Known Gotchas (for agents picking this up) @@ -649,6 +717,8 @@ See the "Annual cycle refresh for a topic" prompt above. - **WikiCFP `` labels**: deadline detail pages use `` for label cells, not ``. The parser looks for `+` pairs. - **Calendar events**: `pa-generate` embeds events as YAML in the `events:` front matter field. The layout at `site/layouts/_default/calendar.html` reads `.Params.events` and initializes FullCalendar. Do not remove the `layout: calendar` front matter field. - **Deadline preservation**: never strip deadline fields from `deadlines.yaml` just because the submission window has closed. Remove an entry only once the conference has taken place. +- **Nothing refreshes deadlines on its own**: `source: manual` entries are preserved across every fetcher run by design, and there is no cron or CI job that would notice. A frozen entry is reported under `stale:`, never fixed. Run Task 7. +- **The calendar's open/elapsed split is a build-time decision**: `pa-generate` compares each cycle against the date it runs, so a round that lapses after the last build stays in the "Conference Deadlines" table until content is regenerated *and committed*. Re-running `./build.sh` periodically is worthwhile even when no deadline data changed. - **WikiCFP IDs are edition-specific**: each year's event gets a new ID. IDs verified at one point in time will be wrong once a new cycle begins. See Task 6 for the annual refresh checklist. --- diff --git a/site/content/cloud-edge/calendar/_index.md b/site/content/cloud-edge/calendar/_index.md index b7864d5..bb0bbdb 100644 --- a/site/content/cloud-edge/calendar/_index.md +++ b/site/content/cloud-edge/calendar/_index.md @@ -1,7 +1,7 @@ --- title: Submission Deadlines layout: calendar -generated: '2026-08-17T18:20:48Z' +generated: '2026-08-17T20:24:13Z' draft: false events: - title: OSDI — abstract deadline @@ -587,24 +587,27 @@ events: | Venue | Name | Cycle | Abstract | Paper deadline | Notification | Event dates | Location | ICORE | |---|---|---|---|---|---|---|---|---| -| [MobiSys](/cloud-edge/venues/conferences/mobisys/) | ACM International Conference on Mobile Systems, Applications, and Services | — | Nov 28, 2025 | Dec 5, 2025 | Mar 2, 2026 | Jun 22-24, 2026 | Cambridge, UK | A | -| [Middleware](/cloud-edge/venues/conferences/middleware/) | ACM/IFIP/USENIX Middleware Conference | Winter | Dec 9, 2025 | Dec 12, 2025 | Mar 6, 2026 | Dec 14-18, 2026 | Tarragona, Spain | A | -| [CCGrid](/cloud-edge/venues/conferences/ccgrid/) | IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing | — | | Dec 21, 2025 | Feb 10, 2026 | May 18-21, 2026 | Sydney, Australia | B | -| [HPDC](/cloud-edge/venues/conferences/hpdc/) | IEEE International Symposium on High Performance Distributed Computing | — | Jan 29, 2026 | Feb 5, 2026 | Mar 31, 2026 | Jul 13, 2026 - Jul 16, 2026 | Cleveland, OH, USA | A | -| [SoCC](/cloud-edge/venues/conferences/socc/) | ACM Symposium on Cloud Computing | Round 1 | Feb 6, 2026 | Feb 13, 2026 | Apr 29, 2026 | Nov 18-20, 2026 | Singapore | — | -| [SOSP](/cloud-edge/venues/conferences/sosp/) | ACM Symposium on Operating Systems Principles | — | Mar 26, 2026 | Apr 1, 2026 | Jul 3, 2026 | Sep 29, 2026 - Oct 2, 2026 | Prague, Czechia | A* | -| [SC](/cloud-edge/venues/conferences/sc/) | International Conference for High Performance Computing, Networking, Storage and Analysis | — | Apr 1, 2026 | Apr 8, 2026 | Jul 1, 2026 | Nov 15-20, 2026 | Chicago, IL, USA | A | -| [NSDI](/cloud-edge/venues/conferences/nsdi/) | USENIX Symposium on Networked Systems Design and Implementation | Spring | Apr 16, 2026 | Apr 23, 2026 | Jul 23, 2026 | May 11-13, 2027 | Providence, RI, USA | National: USA | -| [SEC](/cloud-edge/venues/conferences/sec/) | IEEE/ACM Symposium on Edge Computing | — | | May 8, 2026 | Jul 29, 2026 | Oct 13-16, 2026 | | unranked | -| [EuroSys](/cloud-edge/venues/conferences/eurosys/) | European Conference on Computer Systems | Spring | May 7, 2026 | May 14, 2026 | Aug 21, 2026 | Apr 19-23, 2027 | Rabat, Morocco | A | -| [Middleware](/cloud-edge/venues/conferences/middleware/) | ACM/IFIP/USENIX Middleware Conference | Summer | May 29, 2026 | Jun 5, 2026 | Aug 28, 2026 | Dec 14-18, 2026 | Tarragona, Spain | A | -| [ATC](/cloud-edge/venues/conferences/atc/) | ACM SIGOPS Annual Technical Conference | — | | Jun 10, 2026 | Sep 18, 2026 | Nov 16-18, 2026 | Hong Kong | C | -| [SoCC](/cloud-edge/venues/conferences/socc/) | ACM Symposium on Cloud Computing | Round 2 | Jul 7, 2026 | Jul 14, 2026 | Sep 26, 2026 | Nov 18-20, 2026 | Singapore | — | | [NSDI](/cloud-edge/venues/conferences/nsdi/) | USENIX Symposium on Networked Systems Design and Implementation | Fall | Sep 10, 2026 | Sep 17, 2026 | Dec 8, 2026 | May 11-13, 2027 | Providence, RI, USA | National: USA | | [EuroSys](/cloud-edge/venues/conferences/eurosys/) | European Conference on Computer Systems | Fall | Sep 17, 2026 | Sep 24, 2026 | Jan 29, 2027 | Apr 19-23, 2027 | Rabat, Morocco | A | | [IPDPS](/cloud-edge/venues/conferences/ipdps/) | IEEE International Parallel and Distributed Processing Symposium | — | Oct 1, 2026 | Oct 8, 2026 | Feb 2, 2027 | Jun 1-5, 2027 | Seattle, WA, USA | A | | [OSDI](/cloud-edge/venues/conferences/osdi/) | USENIX Symposium on Operating Systems Design and Implementation | — | Dec 1, 2026 | Dec 8, 2026 | Mar 16, 2027 | Jul 7-9, 2027 | Baltimore, MD, USA | A* | +### Awaiting the next call + +Tracked venues whose recorded rounds have all elapsed — the next CFP has not been published or picked up yet. Dates below are the last round that ran, kept for reference only. + +| Venue | Name | Last recorded deadline | Event dates | Location | ICORE | +|---|---|---|---|---|---| +| [MobiSys](/cloud-edge/venues/conferences/mobisys/) | ACM International Conference on Mobile Systems, Applications, and Services | Dec 5, 2025 | Jun 22-24, 2026 | Cambridge, UK | A | +| [CCGrid](/cloud-edge/venues/conferences/ccgrid/) | IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing | Dec 21, 2025 | May 18-21, 2026 | Sydney, Australia | B | +| [HPDC](/cloud-edge/venues/conferences/hpdc/) | IEEE International Symposium on High Performance Distributed Computing | Feb 5, 2026 | Jul 13, 2026 - Jul 16, 2026 | Cleveland, OH, USA | A | +| [SOSP](/cloud-edge/venues/conferences/sosp/) | ACM Symposium on Operating Systems Principles | Apr 1, 2026 | Sep 29, 2026 - Oct 2, 2026 | Prague, Czechia | A* | +| [SC](/cloud-edge/venues/conferences/sc/) | International Conference for High Performance Computing, Networking, Storage and Analysis | Apr 8, 2026 | Nov 15-20, 2026 | Chicago, IL, USA | A | +| [SEC](/cloud-edge/venues/conferences/sec/) | IEEE/ACM Symposium on Edge Computing | May 8, 2026 | Oct 13-16, 2026 | | unranked | +| [Middleware](/cloud-edge/venues/conferences/middleware/) | ACM/IFIP/USENIX Middleware Conference | Jun 5, 2026 | Dec 14-18, 2026 | Tarragona, Spain | A | +| [ATC](/cloud-edge/venues/conferences/atc/) | ACM SIGOPS Annual Technical Conference | Jun 10, 2026 | Nov 16-18, 2026 | Hong Kong | C | +| [SoCC](/cloud-edge/venues/conferences/socc/) | ACM Symposium on Cloud Computing | Jul 14, 2026 | Nov 18-20, 2026 | Singapore | — | + ## Journal Special Issues No open call at any tracked journal. These journals use rolling submission, so special issue CFPs are their only dated deadlines — and none are recorded right now. diff --git a/site/data/cloud-edge/deadlines.yaml b/site/data/cloud-edge/deadlines.yaml index d7b23a1..b3d94a3 100644 --- a/site/data/cloud-edge/deadlines.yaml +++ b/site/data/cloud-edge/deadlines.yaml @@ -1,4 +1,4 @@ -generated: '2026-08-17T16:18:08Z' +generated: '2026-08-17T20:21:42Z' deadlines: EuroSys: source: manual @@ -115,6 +115,15 @@ deadlines: notification: Feb 10, 2026 camera_ready: Mar 15, 2026 cfp_url: https://ccgrid2026.org/cfp.html + IPDPS: + source: manual + event_dates: Jun 1-5, 2027 + location: Seattle, WA, USA + abstract_deadline: Oct 1, 2026 + submission_deadline: Oct 8, 2026 + notification: Feb 2, 2027 + camera_ready: Feb 20, 2027 + cfp_url: https://www.ipdps.org/ipdps2027/2027-call-for-papers.html SOSP: wikicfp_event_id: '191399' source: wikicfp @@ -126,15 +135,6 @@ deadlines: camera_ready: Aug 28, 2026 cfp_url: https://sigops.org/s/conferences/sosp/2026/cfp.html wikicfp_title: SOSP - IPDPS: - source: manual - event_dates: Jun 1-5, 2027 - location: Seattle, WA, USA - abstract_deadline: Oct 1, 2026 - submission_deadline: Oct 8, 2026 - notification: Feb 2, 2027 - camera_ready: Feb 20, 2027 - cfp_url: https://www.ipdps.org/ipdps2027/2027-call-for-papers.html HPDC: wikicfp_event_id: '191029' source: wikicfp @@ -147,3 +147,40 @@ deadlines: cfp_url: https://hpdc.sci.utah.edu/2026/calls-cfp.html wikicfp_title: HPDC missing: [] +stale: +- acronym: ATC + last_deadline: Jun 10, 2026 + source: manual + cfp_url: https://sigops.org/s/conferences/atc/2026/cfp.html +- acronym: CCGrid + last_deadline: Dec 21, 2025 + source: manual + cfp_url: https://ccgrid2026.org/cfp.html +- acronym: HPDC + last_deadline: Feb 5, 2026 + source: wikicfp + cfp_url: https://hpdc.sci.utah.edu/2026/calls-cfp.html +- acronym: Middleware + last_deadline: Jun 5, 2026 + source: manual + cfp_url: https://middleware-conf.github.io/2026/calls/call-for-research-papers/ +- acronym: MobiSys + last_deadline: Dec 5, 2025 + source: manual + cfp_url: https://www.sigmobile.org/mobisys/2026/ +- acronym: SC + last_deadline: Apr 8, 2026 + source: manual + cfp_url: https://sc26.supercomputing.org/program/papers/ +- acronym: SEC + last_deadline: May 8, 2026 + source: manual + cfp_url: https://acm-ieee-sec.org/2026/ +- acronym: SOSP + last_deadline: Apr 1, 2026 + source: wikicfp + cfp_url: https://sigops.org/s/conferences/sosp/2026/cfp.html +- acronym: SoCC + last_deadline: Jul 14, 2026 + source: manual + cfp_url: https://acmsocc.org/2026/papers.html diff --git a/src/publish_assistant/cycles.py b/src/publish_assistant/cycles.py new file mode 100644 index 0000000..551e4dc --- /dev/null +++ b/src/publish_assistant/cycles.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Submission cycles and their dates. + +Shared by the fetcher and the generator: `fetch_deadlines.py` needs to know +whether a preserved manual entry still describes a future deadline, and +`generate_content.py` needs the same notion of "next" to render venue pages and +the calendar. Keeping one definition here stops the two halves from disagreeing +about which round a reader should act on. +""" +from datetime import datetime + +# Fields describing one submission cycle. Everything else on a deadline entry +# (event_dates, location, cfp_url, source) describes the event itself. +CYCLE_FIELDS = ("name", "abstract_deadline", "submission_deadline", "notification", + "camera_ready") + + +def cycles_of(entry: dict) -> list[dict]: + """The submission rounds an entry describes, without modifying it. + + Venues with a single round stay flat in YAML; this presents both shapes as a + list so callers only ever deal with cycles. Read-only on purpose: the + fetcher writes preserved entries straight back out, so folding a flat entry + in place there would rewrite the file for no reason. + """ + if entry.get("cycles"): + return entry["cycles"] + cycle = {k: entry[k] for k in CYCLE_FIELDS if k in entry} + return [cycle] if cycle else [] + + +def normalise_cycles(entry: dict) -> None: + """Fold a flat, single-cycle entry into a one-element `cycles` list, in place. + + `cfp_url` stays at the top level — it describes the call, not one round — but + a cycle may override it where a venue publishes separate calls per round. + """ + if entry.get("cycles"): + return + cycles = cycles_of(entry) + for field in CYCLE_FIELDS: + entry.pop(field, None) + entry["cycles"] = cycles + + +def parse_date(s: str, month_only: bool = False) -> str | None: + """'Apr 1, 2026' → '2026-04-01', or None if unparseable. + + With month_only, also accepts a bare month and year ('Jan 2027' → + '2027-01-01'). Journal special issues often announce notifications at + month granularity; conference deadlines are always full dates, so this + stays opt-in to keep their output unchanged. + """ + for fmt in ("%b %d, %Y", "%B %d, %Y"): + try: + return datetime.strptime(s.strip(), fmt).strftime("%Y-%m-%d") + except ValueError: + pass + if month_only: + for fmt in ("%b %Y", "%B %Y"): + try: + return datetime.strptime(s.strip(), fmt).strftime("%Y-%m-%d") + except ValueError: + pass + return None + + +def cycle_deadline(cycle: dict) -> str: + """The date a cycle is judged by: its paper deadline, else its abstract deadline.""" + return cycle.get("submission_deadline") or cycle.get("abstract_deadline") or "" + + +def cycle_sort_key(cycle: dict) -> str: + """ISO deadline used to order cycles; unparseable dates sort last.""" + return parse_date(cycle_deadline(cycle)) or "9999-12-31" + + +def open_cycles(entry: dict, today: str | None = None) -> list[dict]: + """Cycles whose paper deadline has not passed, earliest first. + + Empty once every recorded round has elapsed. Callers decide what that means: + a venue page falls back to the last round so it does not go blank, the + calendar sets the venue aside as awaiting its next call, and the fetcher + reports it as a stale manual entry. + """ + today = today or datetime.now().strftime("%Y-%m-%d") + ordered = sorted(cycles_of(entry), key=cycle_sort_key) + return [c for c in ordered if cycle_sort_key(c) >= today] + + +def next_cycle(entry: dict, today: str | None = None) -> dict: + """The cycle a reader should act on. + + The earliest cycle whose paper deadline has not passed; if every cycle has + elapsed, the last one, so a page shows the most recent round rather than + going blank. + """ + ordered = sorted(cycles_of(entry), key=cycle_sort_key) + if not ordered: + return {} + return next(iter(open_cycles(entry, today)), ordered[-1]) diff --git a/src/publish_assistant/fetch_deadlines.py b/src/publish_assistant/fetch_deadlines.py index 4112a6e..0d74e90 100644 --- a/src/publish_assistant/fetch_deadlines.py +++ b/src/publish_assistant/fetch_deadlines.py @@ -10,8 +10,15 @@ For each conference in data/venues.yaml, this script: Conferences where no CFP was found are listed under the 'missing' key in the output YAML — the agent should fill those in manually (Task 2 in README). +Entries whose every recorded round has already elapsed are listed under 'stale'. +Manual entries are preserved untouched (WikiCFP cannot express multi-cycle +venues, so re-fetching them would destroy hand-researched rounds) — reporting is +all this script can do for them. Where the venue's CFP URL carries a year, the +next edition's URL is probed so the report says which ones are worth researching +now. + Usage: - python fetch_deadlines.py [--venues PATH] [--output PATH] + python fetch_deadlines.py [--venues PATH] [--output PATH] [--no-probe] """ import argparse import re @@ -24,6 +31,8 @@ import requests import yaml from bs4 import BeautifulSoup +from publish_assistant.cycles import cycle_deadline, cycles_of, next_cycle, open_cycles + WIKICFP_SEARCH = "http://www.wikicfp.com/cfp/servlet/tool.search" WIKICFP_EVENT = "http://www.wikicfp.com/cfp/servlet/event.showcfp" WIKICFP_SERIES = "http://www.wikicfp.com/cfp/program?id={series_id}" @@ -127,6 +136,87 @@ def best_match(results: list[dict], acronym: str) -> dict | None: return None +# A four-digit year anywhere in the URL ('/2026/', '2027.eurosys.org'), or a +# two-digit one glued to the end of a name ('sc26.supercomputing.org', 'osdi27'). +_YEAR_RE = re.compile(r"(? list[tuple[str, int]]: + """Candidate URLs for the edition after the one `url` describes. + + Venue sites are year-stamped, so the next edition usually lives at the same + URL with the year advanced. Returns (url, year) pairs; empty when the URL + carries no year to bump — HPDC, for instance, moves to a new host each year. + """ + candidates: list[tuple[str, int]] = [] + + years = [int(y) for y in _YEAR_RE.findall(url)] + if years: + # Bump every occurrence: IPDPS carries the year twice in one URL. + bumped = _YEAR_RE.sub(lambda m: str(int(m.group(1)) + 1), url) + candidates.append((bumped, max(years) + 1)) + + short = _SHORT_YEAR_RE.search(url) + if short: + bumped = _SHORT_YEAR_RE.sub(lambda m: f"{int(m.group(1)) + 1:02d}", url) + candidates.append((bumped, 2000 + int(short.group(1)) + 1)) + + return [(u, y) for u, y in candidates if u != url] + + +def probe(session: requests.Session, url: str, year: int) -> bool: + """True when `url` serves a real page for `year`. + + GET rather than HEAD — some venue hosts answer HEAD with 405. The year has + to appear in the body too: a host that serves its current landing page for + any path would otherwise read as next year's site being live. + """ + try: + r = session.get(url, headers=HEADERS, timeout=15, allow_redirects=True) + except requests.RequestException: + return False + return r.status_code == 200 and str(year) in r.text + + +def recorded_year(entry: dict) -> int: + """The latest year this entry already describes. + + A probe result only counts if it is newer than this. Venue URLs sometimes + lag several editions behind — `venues.yaml` still points SC at sc24 — and + bumping one of those by a year lands on a conference that already happened. + """ + dates = [entry.get("event_dates") or ""] + dates += [cycle_deadline(c) for c in cycles_of(entry)] + years = [int(y) for y in _YEAR_RE.findall(" ".join(dates))] + return max(years, default=0) + + +def find_next_edition(session: requests.Session, entry: dict, conf: dict) -> str | None: + """URL of the next edition's call, if it is already online.""" + floor = recorded_year(entry) + for source in (entry.get("cfp_url"), conf.get("url")): + for url, year in next_edition_urls(source or ""): + if year > floor and probe(session, url, year): + return url + return None + + +def stale_record(session: requests.Session, acronym: str, entry: dict, conf: dict, + probe_enabled: bool) -> dict: + """Describe an entry whose every recorded round has elapsed.""" + record = {"acronym": acronym, + "last_deadline": cycle_deadline(next_cycle(entry)) or "TBD", + "source": entry.get("source") or "unknown"} + if entry.get("cfp_url"): + record["cfp_url"] = entry["cfp_url"] + if probe_enabled: + nxt = find_next_edition(session, entry, conf) + if nxt: + record["next_edition_url"] = nxt + return record + + def process_venue(session: requests.Session, conf: dict) -> dict | None: """Return deadline data for a conference, or None if not found.""" acronym = conf.get("acronym", "") @@ -157,6 +247,8 @@ def main() -> None: ap = argparse.ArgumentParser(description="Fetch submission deadlines from WikiCFP.") ap.add_argument("--venues", default="site/data/cloud-edge/venues.yaml") ap.add_argument("--output", default="site/data/cloud-edge/deadlines.yaml") + ap.add_argument("--no-probe", action="store_true", + help="skip the HTTP check for a stale venue's next-edition page") args = ap.parse_args() venues_path = Path(args.venues) @@ -185,12 +277,23 @@ def main() -> None: session = requests.Session() found: dict[str, dict] = dict(existing) # start with manual entries missing: list[str] = [] + stale: list[dict] = [] + probe_enabled = not args.no_probe for conf in conferences: acronym = conf.get("acronym", "") print(f" {acronym} …", file=sys.stderr) if acronym in existing: - print(f" manual entry preserved", file=sys.stderr) + # Preserved verbatim either way — all this run can do is say whether + # the entry still describes a deadline anyone can act on. + entry = existing[acronym] + if open_cycles(entry): + print(" manual entry preserved", file=sys.stderr) + else: + record = stale_record(session, acronym, entry, conf, probe_enabled) + stale.append(record) + print(f" manual entry preserved — STALE since {record['last_deadline']}", + file=sys.stderr) continue try: time.sleep(REQUEST_DELAY) @@ -199,6 +302,11 @@ def main() -> None: found[acronym] = data dl = data.get("submission_deadline") or data.get("abstract_deadline") or "?" print(f" deadline: {dl}", file=sys.stderr) + # WikiCFP IDs are edition-specific, so a pinned ID keeps serving + # a conference that already happened. Same symptom, same report. + if not open_cycles(data): + stale.append(stale_record(session, acronym, data, conf, probe_enabled)) + print(f" STALE — this edition has already run", file=sys.stderr) else: missing.append(acronym) print(f" not found on WikiCFP", file=sys.stderr) @@ -206,10 +314,14 @@ def main() -> None: missing.append(acronym) print(f" error: {exc}", file=sys.stderr) + # Actionable first: venues whose next edition is already online. + stale.sort(key=lambda r: ("next_edition_url" not in r, r["acronym"])) + output = { "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "deadlines": found, "missing": missing, + "stale": stale, } out_path = Path(args.output) @@ -218,7 +330,8 @@ def main() -> None: yaml.dump(output, f, allow_unicode=True, sort_keys=False, default_flow_style=False) print( - f"\nResults: {len(found)} found, {len(missing)} missing → {out_path}", + f"\nResults: {len(found)} found, {len(missing)} missing, {len(stale)} stale " + f"→ {out_path}", file=sys.stderr, ) if missing: @@ -226,6 +339,18 @@ def main() -> None: f"Missing (fill in manually — see README Task 2): {', '.join(missing)}", file=sys.stderr, ) + if stale: + print("\nStale — every recorded round has elapsed (see README Task 7):", + file=sys.stderr) + for record in stale: + if record.get("next_edition_url"): + note = f"NEW EDITION: {record['next_edition_url']}" + elif probe_enabled: + note = "next edition not announced yet" + else: + note = "next edition not checked (--no-probe)" + print(f" {record['acronym']:<12} last deadline " + f"{record['last_deadline']:<16} {note}", file=sys.stderr) if __name__ == "__main__": diff --git a/src/publish_assistant/generate_content.py b/src/publish_assistant/generate_content.py index 1c1da34..a7ea894 100644 --- a/src/publish_assistant/generate_content.py +++ b/src/publish_assistant/generate_content.py @@ -30,6 +30,14 @@ from pathlib import Path import yaml +from publish_assistant.cycles import ( + cycle_deadline, + next_cycle, + normalise_cycles, + open_cycles, + parse_date, +) + PRESERVED_VENUE_FIELDS = {"notes", "deadline_source"} # Fields removed from existing front matter on every regeneration: @@ -120,26 +128,6 @@ def scimago_lookup(data: dict[str, dict], full_name: str, issn: str) -> dict: return {} -# Fields describing one submission cycle. Everything else on a deadline entry -# (event_dates, location, cfp_url, source) describes the event itself. -CYCLE_FIELDS = ("name", "abstract_deadline", "submission_deadline", "notification", - "camera_ready") - - -def _normalise_cycles(entry: dict) -> None: - """Fold a flat, single-cycle entry into a one-element `cycles` list, in place. - - Venues with a single submission round stay flat in YAML; consumers below only - ever see `cycles`. `cfp_url` stays at the top level — it describes the call, - not one round — but a cycle may override it where a venue publishes separate - calls per round. - """ - if entry.get("cycles"): - return - cycle = {k: entry.pop(k) for k in CYCLE_FIELDS if k in entry} - entry["cycles"] = [cycle] if cycle else [] - - def load_deadlines(path: str) -> dict[str, dict]: p = Path(path) if not p.exists(): @@ -149,7 +137,7 @@ def load_deadlines(path: str) -> dict[str, dict]: deadlines = data.get("deadlines") or {} for entry in deadlines.values(): if entry: - _normalise_cycles(entry) + normalise_cycles(entry) return deadlines @@ -353,52 +341,6 @@ def _journal_body(fm: dict, digests: list[dict] | None = None, base_path: str = # Date helpers # --------------------------------------------------------------------------- -def _parse_date(s: str, month_only: bool = False) -> str | None: - """'Apr 1, 2026' → '2026-04-01', or None if unparseable. - - With month_only, also accepts a bare month and year ('Jan 2027' → - '2027-01-01'). Journal special issues often announce notifications at - month granularity; conference deadlines are always full dates, so this - stays opt-in to keep their output unchanged. - """ - for fmt in ("%b %d, %Y", "%B %d, %Y"): - try: - return datetime.strptime(s.strip(), fmt).strftime("%Y-%m-%d") - except ValueError: - pass - if month_only: - for fmt in ("%b %Y", "%B %Y"): - try: - return datetime.strptime(s.strip(), fmt).strftime("%Y-%m-%d") - except ValueError: - pass - return None - - -def _cycle_deadline(cycle: dict) -> str: - """The date a cycle is judged by: its paper deadline, else its abstract deadline.""" - return cycle.get("submission_deadline") or cycle.get("abstract_deadline") or "" - - -def _next_cycle(entry: dict, today: str | None = None) -> dict: - """The cycle a reader should act on. - - The earliest cycle whose paper deadline has not passed; if every cycle has - elapsed, the last one, so a page shows the most recent round rather than - going blank. Cycles with unparseable dates sort last. - """ - cycles = entry.get("cycles") or [] - if not cycles: - return {} - today = today or datetime.now().strftime("%Y-%m-%d") - ordered = sorted(cycles, key=lambda c: _parse_date(_cycle_deadline(c)) or "9999-12-31") - for cycle in ordered: - iso = _parse_date(_cycle_deadline(cycle)) - if iso and iso >= today: - return cycle - return ordered[-1] - - def _parse_date_range(s: str) -> tuple[str | None, str | None]: """Return (start_iso, end_iso_exclusive) from a date-range string. @@ -410,22 +352,22 @@ def _parse_date_range(s: str) -> tuple[str | None, str | None]: s = s.strip() if " - " in s: left, right = s.split(" - ", 1) - start = _parse_date(left) - end_s = _parse_date(right) + start = parse_date(left) + end_s = parse_date(right) if start and end_s: end_dt = datetime.strptime(end_s, "%Y-%m-%d") + timedelta(days=1) return start, end_dt.strftime("%Y-%m-%d") m = re.match(r"(\w+)\s+(\d+)-(\d+),\s+(\d{4})", s) if m: month, d1, d2, year = m.groups() - start = _parse_date(f"{month} {d1}, {year}") + start = parse_date(f"{month} {d1}, {year}") try: end_dt = datetime.strptime(f"{month} {d2}, {year}", "%b %d, %Y") + timedelta(days=1) if start: return start, end_dt.strftime("%Y-%m-%d") except ValueError: pass - return _parse_date(s), None + return parse_date(s), None # --------------------------------------------------------------------------- @@ -455,7 +397,7 @@ def _conferences_section_body(venues: dict, icore: dict, deadlines: dict, dl = deadlines.get(acronym, {}) # The venue index lists venues, not cycles: show the next open round and # name it, so a passed round never reads as "you missed this conference". - nxt = _next_cycle(dl) + nxt = next_cycle(dl) abstract = nxt.get("abstract_deadline") or "" paper = nxt.get("submission_deadline") or nxt.get("abstract_deadline") or "TBD" if len(dl.get("cycles") or []) > 1 and nxt.get("name"): @@ -539,7 +481,7 @@ def _calendar_events(venues: dict, deadlines: dict, base_path: str = "") -> list ("notification", "notification", "notification"), ]: color, text_color = EVENT_COLORS[kind] - iso = _parse_date(cycle.get(field) or "") + iso = parse_date(cycle.get(field) or "") if iso: ev: dict = {"title": f"{prefix} — {label}", "start": iso, "url": url, "color": color, @@ -580,7 +522,7 @@ def _calendar_events(venues: dict, deadlines: dict, base_path: str = "") -> list ("notification", "notification"), ]: # Notifications are often month-only ('Jan 2027'); deadlines are not. - iso = _parse_date(si.get(field) or "", month_only=(field == "notification")) + iso = parse_date(si.get(field) or "", month_only=(field == "notification")) if iso: ev = {"title": f"{acronym} SI — {si_title} — {label}", "start": iso, "url": url, "color": journal_color, @@ -603,14 +545,26 @@ def _calendar_body(venues: dict, icore: dict, deadlines: dict, base_path: str = lines: list[str] = [] conf_rows = [] + stale_rows = [] for conf in venues.get("conferences") or []: acronym = conf.get("acronym", "") rank = icore.get(acronym.upper(), "—") or "—" dl_data = deadlines.get(acronym, {}) event = dl_data.get("event_dates") or "—" location = dl_data.get("location") or "" - # One row per submission cycle — a venue's later rounds are the point. - for cycle in dl_data.get("cycles") or [{}]: + # One row per still-open submission cycle — a venue's later rounds are + # the point, but an elapsed round is not something a reader can act on, + # and sorting by deadline parks it at the top of the table. + still_open = open_cycles(dl_data) + if not still_open: + # Every recorded round has passed: the venue is still tracked, its + # next CFP just isn't out (or isn't fetched) yet. Listed separately + # below so a stale date never reads as an upcoming deadline. + last = next_cycle(dl_data) + stale_rows.append((acronym, conf.get("full_name", ""), + cycle_deadline(last) or "TBD", event, location, rank)) + continue + for cycle in still_open: abstract = cycle.get("abstract_deadline") or "" paper = cycle.get("submission_deadline") or cycle.get("abstract_deadline") or "TBD" notif = cycle.get("notification") or "" @@ -618,19 +572,38 @@ def _calendar_body(venues: dict, icore: dict, deadlines: dict, base_path: str = abstract, paper, notif, event, location, rank)) conf_rows.sort(key=lambda r: (parse_deadline(r[4]), r[0])) + stale_rows.sort(key=lambda r: (parse_deadline(r[2]), r[0])) - lines += [ - "\n## Conference Deadlines\n", - "| Venue | Name | Cycle | Abstract | Paper deadline | Notification | Event dates | Location | ICORE |", - "|---|---|---|---|---|---|---|---|---|", - ] - for acronym, full_name, cycle_name, abstract, paper, notif, event, location, rank in conf_rows: - abs_cell = abstract if (abstract and abstract != paper) else "" - link = f"[{acronym}]({base_path}/venues/conferences/{acronym.lower()}/)" - lines.append( - f"| {link} | {full_name} | {cycle_name} | {abs_cell} | {paper} | {notif} " - f"| {event} | {location} | {rank} |" - ) + lines.append("\n## Conference Deadlines\n") + if conf_rows: + lines += [ + "| Venue | Name | Cycle | Abstract | Paper deadline | Notification | Event dates | Location | ICORE |", + "|---|---|---|---|---|---|---|---|---|", + ] + for acronym, full_name, cycle_name, abstract, paper, notif, event, location, rank in conf_rows: + abs_cell = abstract if (abstract and abstract != paper) else "" + link = f"[{acronym}]({base_path}/venues/conferences/{acronym.lower()}/)" + lines.append( + f"| {link} | {full_name} | {cycle_name} | {abs_cell} | {paper} | {notif} " + f"| {event} | {location} | {rank} |" + ) + else: + lines.append("No open call at any tracked conference.") + + if stale_rows: + lines += [ + "\n### Awaiting the next call\n", + "Tracked venues whose recorded rounds have all elapsed — the next CFP " + "has not been published or picked up yet. Dates below are the last " + "round that ran, kept for reference only.\n", + "| Venue | Name | Last recorded deadline | Event dates | Location | ICORE |", + "|---|---|---|---|---|---|", + ] + for acronym, full_name, last_deadline, event, location, rank in stale_rows: + link = f"[{acronym}]({base_path}/venues/conferences/{acronym.lower()}/)" + lines.append( + f"| {link} | {full_name} | {last_deadline} | {event} | {location} | {rank} |" + ) si_rows = [] for journal in venues.get("journals") or []: @@ -732,7 +705,7 @@ def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path, } dl = deadlines.get(acronym, {}) cycles = dl.get("cycles") or [] - nxt = _next_cycle(dl) + nxt = next_cycle(dl) if nxt.get("submission_deadline"): new_fm["next_deadline"] = nxt["submission_deadline"] elif nxt.get("abstract_deadline"):