Initial commit
Hugo/PaperMod static site tracking 13 conferences and 7 journals for edge and cloud systems research. Includes FullCalendar deadline view, ICORE/SCImago rankings, DBLP paper digest pipeline, and Python fetch/generate scripts. PaperMod added as a git submodule.
This commit is contained in:
khannurien
232
src/publish_assistant/fetch_deadlines.py
Normal file
232
src/publish_assistant/fetch_deadlines.py
Normal file
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch submission deadlines for tracked conferences from WikiCFP.
|
||||
|
||||
For each conference in data/venues.yaml, this script:
|
||||
1. Searches WikiCFP by acronym (or uses wikicfp_series if provided)
|
||||
2. Scrapes the matching CFP page for deadline dates
|
||||
3. Writes results to data/deadlines.yaml
|
||||
|
||||
Conferences where no CFP was found are listed under the 'missing' key in the
|
||||
output YAML — the agent should fill those in manually (Task 2 in README).
|
||||
|
||||
Usage:
|
||||
python fetch_deadlines.py [--venues PATH] [--output PATH]
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
WIKICFP_SEARCH = "http://www.wikicfp.com/cfp/servlet/tool.search"
|
||||
WIKICFP_EVENT = "http://www.wikicfp.com/cfp/servlet/event.showcfp"
|
||||
WIKICFP_SERIES = "http://www.wikicfp.com/cfp/program?id={series_id}"
|
||||
HEADERS = {
|
||||
"User-Agent": "publish-assistant/1.0 (academic research tool)",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
REQUEST_DELAY = 2.5
|
||||
|
||||
|
||||
def search_wikicfp(session: requests.Session, query: str) -> list[dict]:
|
||||
"""Search WikiCFP for a query string. Returns a list of CFP summary dicts."""
|
||||
r = session.get(
|
||||
WIKICFP_SEARCH,
|
||||
params={"q": query, "year": "f"},
|
||||
headers=HEADERS,
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
results = []
|
||||
|
||||
# WikiCFP search results use paired rows: [name+link | when/where] [desc]
|
||||
contsec = soup.find("div", class_="contsec")
|
||||
if not contsec:
|
||||
return results
|
||||
|
||||
rows = contsec.find_all("tr")
|
||||
i = 0
|
||||
while i < len(rows):
|
||||
cells = rows[i].find_all("td")
|
||||
# First row of a pair has 4 cells: title, acronym, deadline, event-date
|
||||
if len(cells) >= 3:
|
||||
link = cells[0].find("a", href=re.compile(r"showcfp\?eventid="))
|
||||
if link:
|
||||
m = re.search(r"eventid=(\d+)", link["href"])
|
||||
event_id = m.group(1) if m else None
|
||||
results.append({
|
||||
"event_id": event_id,
|
||||
"title": cells[0].get_text(strip=True),
|
||||
"acronym": cells[1].get_text(strip=True) if len(cells) > 1 else "",
|
||||
"deadline": cells[2].get_text(strip=True) if len(cells) > 2 else "",
|
||||
"event_dates": cells[3].get_text(strip=True) if len(cells) > 3 else "",
|
||||
})
|
||||
i += 1
|
||||
return results
|
||||
|
||||
|
||||
def fetch_cfp_details(session: requests.Session, event_id: str) -> dict:
|
||||
"""Scrape a WikiCFP event page for structured deadline information."""
|
||||
r = session.get(
|
||||
WIKICFP_EVENT,
|
||||
params={"eventid": event_id},
|
||||
headers=HEADERS,
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
|
||||
details: dict = {"wikicfp_event_id": event_id, "source": "wikicfp"}
|
||||
|
||||
# WikiCFP event pages use <th> for labels and <td> for values in the same <tr>
|
||||
for row in soup.find_all("tr"):
|
||||
th = row.find("th", recursive=False)
|
||||
td = row.find("td", recursive=False)
|
||||
if not (th and td):
|
||||
continue
|
||||
label = th.get_text(strip=True).lower()
|
||||
value = td.get_text(" ", strip=True)
|
||||
if not value or value in ("N/A", "TBD"):
|
||||
continue
|
||||
if "abstract" in label:
|
||||
details["abstract_deadline"] = value
|
||||
elif "submission" in label or "paper" in label:
|
||||
details["submission_deadline"] = value
|
||||
elif "notification" in label:
|
||||
details["notification"] = value
|
||||
elif "camera" in label or "final" in label:
|
||||
details["camera_ready"] = value
|
||||
elif "when" in label:
|
||||
details["event_dates"] = value
|
||||
elif "where" in label or "location" in label:
|
||||
details["location"] = value
|
||||
|
||||
# Grab CFP link from the page header area
|
||||
cfp_link = soup.find("a", href=re.compile(r"^https?://"), string=re.compile(r"cfp|call|submit", re.I))
|
||||
if cfp_link and "cfp_url" not in details:
|
||||
details["cfp_url"] = cfp_link["href"]
|
||||
|
||||
return details
|
||||
|
||||
|
||||
def best_match(results: list[dict], acronym: str) -> dict | None:
|
||||
"""Return the most likely match for a conference acronym from search results."""
|
||||
acronym_up = acronym.upper()
|
||||
# Require the acronym to appear as a standalone token (e.g. "SOSP 2026", not "EESP-SC")
|
||||
pattern = re.compile(r"(?<![A-Z-])" + re.escape(acronym_up) + r"(?![A-Z])")
|
||||
for r in results:
|
||||
if pattern.search(r["title"].upper()):
|
||||
return r
|
||||
return None
|
||||
|
||||
|
||||
def process_venue(session: requests.Session, conf: dict) -> dict | None:
|
||||
"""Return deadline data for a conference, or None if not found."""
|
||||
acronym = conf.get("acronym", "")
|
||||
wikicfp_id = conf.get("wikicfp_id")
|
||||
|
||||
# wikicfp_id: false means explicitly not on WikiCFP (skip search)
|
||||
if wikicfp_id is False:
|
||||
return None
|
||||
|
||||
# wikicfp_id: "<number>" means use that event page directly
|
||||
if wikicfp_id:
|
||||
details = fetch_cfp_details(session, str(wikicfp_id))
|
||||
details["wikicfp_title"] = acronym
|
||||
return details
|
||||
|
||||
results = search_wikicfp(session, acronym)
|
||||
match = best_match(results, acronym)
|
||||
if not match or not match.get("event_id"):
|
||||
return None
|
||||
|
||||
time.sleep(REQUEST_DELAY)
|
||||
details = fetch_cfp_details(session, match["event_id"])
|
||||
details["wikicfp_title"] = match.get("title", "")
|
||||
return details
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Fetch submission deadlines from WikiCFP.")
|
||||
ap.add_argument("--venues", default="site/data/venues.yaml")
|
||||
ap.add_argument("--output", default="site/data/deadlines.yaml")
|
||||
args = ap.parse_args()
|
||||
|
||||
venues_path = Path(args.venues)
|
||||
if not venues_path.exists():
|
||||
print(f"Error: {venues_path} not found. Populate data/venues.yaml first.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with open(venues_path) as f:
|
||||
venues = yaml.safe_load(f) or {}
|
||||
|
||||
conferences = venues.get("conferences") or []
|
||||
if not conferences:
|
||||
print("No conferences in venues.yaml. Nothing to do.", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
# Preserve any manually-entered deadlines from the previous output
|
||||
existing: dict[str, dict] = {}
|
||||
out_path = Path(args.output)
|
||||
if out_path.exists():
|
||||
with open(out_path) as f:
|
||||
prev = yaml.safe_load(f) or {}
|
||||
for acronym, data in (prev.get("deadlines") or {}).items():
|
||||
if (data or {}).get("source") == "manual":
|
||||
existing[acronym] = data
|
||||
|
||||
session = requests.Session()
|
||||
found: dict[str, dict] = dict(existing) # start with manual entries
|
||||
missing: list[str] = []
|
||||
|
||||
for conf in conferences:
|
||||
acronym = conf.get("acronym", "")
|
||||
print(f" {acronym} …", file=sys.stderr)
|
||||
if acronym in existing:
|
||||
print(f" manual entry preserved", file=sys.stderr)
|
||||
continue
|
||||
try:
|
||||
time.sleep(REQUEST_DELAY)
|
||||
data = process_venue(session, conf)
|
||||
if data:
|
||||
found[acronym] = data
|
||||
dl = data.get("submission_deadline") or data.get("abstract_deadline") or "?"
|
||||
print(f" deadline: {dl}", file=sys.stderr)
|
||||
else:
|
||||
missing.append(acronym)
|
||||
print(f" not found on WikiCFP", file=sys.stderr)
|
||||
except Exception as exc:
|
||||
missing.append(acronym)
|
||||
print(f" error: {exc}", file=sys.stderr)
|
||||
|
||||
output = {
|
||||
"generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"deadlines": found,
|
||||
"missing": missing,
|
||||
}
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(out_path, "w") as f:
|
||||
yaml.dump(output, f, allow_unicode=True, sort_keys=False, default_flow_style=False)
|
||||
|
||||
print(
|
||||
f"\nResults: {len(found)} found, {len(missing)} missing → {out_path}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if missing:
|
||||
print(
|
||||
f"Missing (fill in manually — see README Task 2): {', '.join(missing)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user