publish-assistant/src/publish_assistant/fetch_deadlines.py

#!/usr/bin/env python3
"""
Fetch submission deadlines for tracked conferences from WikiCFP.

For each conference in data/venues.yaml, this script:
  1. Searches WikiCFP by acronym (or uses wikicfp_series if provided)
  2. Scrapes the matching CFP page for deadline dates
  3. Writes results to data/deadlines.yaml

Conferences where no CFP was found are listed under the 'missing' key in the
output YAML — the agent should fill those in manually (Task 2 in README).

Entries whose every recorded round has already elapsed are listed under 'stale'.
Manual entries are preserved untouched (WikiCFP cannot express multi-cycle
venues, so re-fetching them would destroy hand-researched rounds) — reporting is
all this script can do for them. Where the venue's CFP URL carries a year, the
next edition's URL is probed so the report says which ones are worth researching
now.

Usage:
    python fetch_deadlines.py [--venues PATH] [--output PATH] [--no-probe]
"""
import argparse
import re
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

import requests
import yaml
from bs4 import BeautifulSoup

from publish_assistant.cycles import cycle_deadline, cycles_of, next_cycle, open_cycles

WIKICFP_SEARCH = "http://www.wikicfp.com/cfp/servlet/tool.search"
WIKICFP_EVENT = "http://www.wikicfp.com/cfp/servlet/event.showcfp"
WIKICFP_SERIES = "http://www.wikicfp.com/cfp/program?id={series_id}"
HEADERS = {
    "User-Agent": "publish-assistant/1.0 (academic research tool)",
    "Accept-Language": "en-US,en;q=0.9",
}
REQUEST_DELAY = 2.5


def search_wikicfp(session: requests.Session, query: str) -> list[dict]:
    """Search WikiCFP for a query string. Returns a list of CFP summary dicts."""
    r = session.get(
        WIKICFP_SEARCH,
        params={"q": query, "year": "f"},
        headers=HEADERS,
        timeout=30,
    )
    r.raise_for_status()
    soup = BeautifulSoup(r.text, "lxml")
    results = []

    # WikiCFP search results use paired rows: [name+link | when/where] [desc]
    contsec = soup.find("div", class_="contsec")
    if not contsec:
        return results

    rows = contsec.find_all("tr")
    i = 0
    while i < len(rows):
        cells = rows[i].find_all("td")
        # First row of a pair has 4 cells: title, acronym, deadline, event-date
        if len(cells) >= 3:
            link = cells[0].find("a", href=re.compile(r"showcfp\?eventid="))
            if link:
                m = re.search(r"eventid=(\d+)", link["href"])
                event_id = m.group(1) if m else None
                results.append({
                    "event_id": event_id,
                    "title": cells[0].get_text(strip=True),
                    "acronym": cells[1].get_text(strip=True) if len(cells) > 1 else "",
                    "deadline": cells[2].get_text(strip=True) if len(cells) > 2 else "",
                    "event_dates": cells[3].get_text(strip=True) if len(cells) > 3 else "",
                })
        i += 1
    return results


def fetch_cfp_details(session: requests.Session, event_id: str) -> dict:
    """Scrape a WikiCFP event page for structured deadline information."""
    r = session.get(
        WIKICFP_EVENT,
        params={"eventid": event_id},
        headers=HEADERS,
        timeout=30,
    )
    r.raise_for_status()
    soup = BeautifulSoup(r.text, "lxml")

    details: dict = {"wikicfp_event_id": event_id, "source": "wikicfp"}

    # WikiCFP event pages use <th> for labels and <td> for values in the same <tr>
    for row in soup.find_all("tr"):
        th = row.find("th", recursive=False)
        td = row.find("td", recursive=False)
        if not (th and td):
            continue
        label = th.get_text(strip=True).lower()
        value = td.get_text(" ", strip=True)
        if not value or value in ("N/A", "TBD"):
            continue
        if "abstract" in label:
            details["abstract_deadline"] = value
        elif "submission" in label or "paper" in label:
            details["submission_deadline"] = value
        elif "notification" in label:
            details["notification"] = value
        elif "camera" in label or "final" in label:
            details["camera_ready"] = value
        elif "when" in label:
            details["event_dates"] = value
        elif "where" in label or "location" in label:
            details["location"] = value

    # Grab CFP link from the page header area
    cfp_link = soup.find("a", href=re.compile(r"^https?://"), string=re.compile(r"cfp|call|submit", re.I))
    if cfp_link and "cfp_url" not in details:
        details["cfp_url"] = cfp_link["href"]

    return details


def best_match(results: list[dict], acronym: str) -> dict | None:
    """Return the most likely match for a conference acronym from search results."""
    acronym_up = acronym.upper()
    # Require the acronym to appear as a standalone token (e.g. "SOSP 2026", not "EESP-SC")
    pattern = re.compile(r"(?<![A-Z-])" + re.escape(acronym_up) + r"(?![A-Z])")
    for r in results:
        if pattern.search(r["title"].upper()):
            return r
    return None


# A four-digit year anywhere in the URL ('/2026/', '2027.eurosys.org'), or a
# two-digit one glued to the end of a name ('sc26.supercomputing.org', 'osdi27').
_YEAR_RE = re.compile(r"(?<!\d)(20\d{2})(?!\d)")
_SHORT_YEAR_RE = re.compile(r"(?<=[a-z])(\d{2})(?!\d)", re.I)


def next_edition_urls(url: str) -> list[tuple[str, int]]:
    """Candidate URLs for the edition after the one `url` describes.

    Venue sites are year-stamped, so the next edition usually lives at the same
    URL with the year advanced. Returns (url, year) pairs; empty when the URL
    carries no year to bump — HPDC, for instance, moves to a new host each year.
    """
    candidates: list[tuple[str, int]] = []

    years = [int(y) for y in _YEAR_RE.findall(url)]
    if years:
        # Bump every occurrence: IPDPS carries the year twice in one URL.
        bumped = _YEAR_RE.sub(lambda m: str(int(m.group(1)) + 1), url)
        candidates.append((bumped, max(years) + 1))

    short = _SHORT_YEAR_RE.search(url)
    if short:
        bumped = _SHORT_YEAR_RE.sub(lambda m: f"{int(m.group(1)) + 1:02d}", url)
        candidates.append((bumped, 2000 + int(short.group(1)) + 1))

    return [(u, y) for u, y in candidates if u != url]


def probe(session: requests.Session, url: str, year: int) -> bool:
    """True when `url` serves a real page for `year`.

    GET rather than HEAD — some venue hosts answer HEAD with 405. The year has
    to appear in the body too: a host that serves its current landing page for
    any path would otherwise read as next year's site being live.
    """
    try:
        r = session.get(url, headers=HEADERS, timeout=15, allow_redirects=True)
    except requests.RequestException:
        return False
    return r.status_code == 200 and str(year) in r.text


def recorded_year(entry: dict) -> int:
    """The latest year this entry already describes.

    A probe result only counts if it is newer than this. Venue URLs sometimes
    lag several editions behind — `venues.yaml` still points SC at sc24 — and
    bumping one of those by a year lands on a conference that already happened.
    """
    dates = [entry.get("event_dates") or ""]
    dates += [cycle_deadline(c) for c in cycles_of(entry)]
    years = [int(y) for y in _YEAR_RE.findall(" ".join(dates))]
    return max(years, default=0)


def find_next_edition(session: requests.Session, entry: dict, conf: dict) -> str | None:
    """URL of the next edition's call, if it is already online."""
    floor = recorded_year(entry)
    for source in (entry.get("cfp_url"), conf.get("url")):
        for url, year in next_edition_urls(source or ""):
            if year > floor and probe(session, url, year):
                return url
    return None


def stale_record(session: requests.Session, acronym: str, entry: dict, conf: dict,
                 probe_enabled: bool) -> dict:
    """Describe an entry whose every recorded round has elapsed."""
    record = {"acronym": acronym,
              "last_deadline": cycle_deadline(next_cycle(entry)) or "TBD",
              "source": entry.get("source") or "unknown"}
    if entry.get("cfp_url"):
        record["cfp_url"] = entry["cfp_url"]
    if probe_enabled:
        nxt = find_next_edition(session, entry, conf)
        if nxt:
            record["next_edition_url"] = nxt
    return record


def process_venue(session: requests.Session, conf: dict) -> dict | None:
    """Return deadline data for a conference, or None if not found."""
    acronym = conf.get("acronym", "")
    wikicfp_id = conf.get("wikicfp_id")

    # wikicfp_id: false means explicitly not on WikiCFP (skip search)
    if wikicfp_id is False:
        return None

    # wikicfp_id: "<number>" means use that event page directly
    if wikicfp_id:
        details = fetch_cfp_details(session, str(wikicfp_id))
        details["wikicfp_title"] = acronym
        return details

    results = search_wikicfp(session, acronym)
    match = best_match(results, acronym)
    if not match or not match.get("event_id"):
        return None

    time.sleep(REQUEST_DELAY)
    details = fetch_cfp_details(session, match["event_id"])
    details["wikicfp_title"] = match.get("title", "")
    return details


def main() -> None:
    ap = argparse.ArgumentParser(description="Fetch submission deadlines from WikiCFP.")
    ap.add_argument("--venues", default="site/data/cloud-edge/venues.yaml")
    ap.add_argument("--output", default="site/data/cloud-edge/deadlines.yaml")
    ap.add_argument("--no-probe", action="store_true",
                    help="skip the HTTP check for a stale venue's next-edition page")
    args = ap.parse_args()

    venues_path = Path(args.venues)
    if not venues_path.exists():
        print(f"Error: {venues_path} not found. Populate data/venues.yaml first.", file=sys.stderr)
        sys.exit(1)

    with open(venues_path) as f:
        venues = yaml.safe_load(f) or {}

    conferences = venues.get("conferences") or []
    if not conferences:
        print("No conferences in venues.yaml. Nothing to do.", file=sys.stderr)
        sys.exit(0)

    # Preserve any manually-entered deadlines from the previous output
    existing: dict[str, dict] = {}
    out_path = Path(args.output)
    if out_path.exists():
        with open(out_path) as f:
            prev = yaml.safe_load(f) or {}
        for acronym, data in (prev.get("deadlines") or {}).items():
            if (data or {}).get("source") == "manual":
                existing[acronym] = data

    session = requests.Session()
    found: dict[str, dict] = dict(existing)  # start with manual entries
    missing: list[str] = []
    stale: list[dict] = []
    probe_enabled = not args.no_probe

    for conf in conferences:
        acronym = conf.get("acronym", "")
        print(f"  {acronym} …", file=sys.stderr)
        if acronym in existing:
            # Preserved verbatim either way — all this run can do is say whether
            # the entry still describes a deadline anyone can act on.
            entry = existing[acronym]
            if open_cycles(entry):
                print("    manual entry preserved", file=sys.stderr)
            else:
                record = stale_record(session, acronym, entry, conf, probe_enabled)
                stale.append(record)
                print(f"    manual entry preserved — STALE since {record['last_deadline']}",
                      file=sys.stderr)
            continue
        try:
            time.sleep(REQUEST_DELAY)
            data = process_venue(session, conf)
            if data:
                found[acronym] = data
                dl = data.get("submission_deadline") or data.get("abstract_deadline") or "?"
                print(f"    deadline: {dl}", file=sys.stderr)
                # WikiCFP IDs are edition-specific, so a pinned ID keeps serving
                # a conference that already happened. Same symptom, same report.
                if not open_cycles(data):
                    stale.append(stale_record(session, acronym, data, conf, probe_enabled))
                    print(f"    STALE — this edition has already run", file=sys.stderr)
            else:
                missing.append(acronym)
                print(f"    not found on WikiCFP", file=sys.stderr)
        except Exception as exc:
            missing.append(acronym)
            print(f"    error: {exc}", file=sys.stderr)

    # Actionable first: venues whose next edition is already online.
    stale.sort(key=lambda r: ("next_edition_url" not in r, r["acronym"]))

    output = {
        "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "deadlines": found,
        "missing": missing,
        "stale": stale,
    }

    out_path = Path(args.output)
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with open(out_path, "w") as f:
        yaml.dump(output, f, allow_unicode=True, sort_keys=False, default_flow_style=False)

    print(
        f"\nResults: {len(found)} found, {len(missing)} missing, {len(stale)} stale "
        f"→ {out_path}",
        file=sys.stderr,
    )
    if missing:
        print(
            f"Missing (fill in manually — see README Task 2): {', '.join(missing)}",
            file=sys.stderr,
        )
    if stale:
        print("\nStale — every recorded round has elapsed (see README Task 7):",
              file=sys.stderr)
        for record in stale:
            if record.get("next_edition_url"):
                note = f"NEW EDITION: {record['next_edition_url']}"
            elif probe_enabled:
                note = "next edition not announced yet"
            else:
                note = "next edition not checked (--no-probe)"
            print(f"  {record['acronym']:<12} last deadline "
                  f"{record['last_deadline']:<16} {note}", file=sys.stderr)


if __name__ == "__main__":
    main()