vincent/publish-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

content updates, various fixes

Browse Source
This commit is contained in:
khannurien
2026-04-26 12:57:40 +00:00
parent 8484abea47
commit 1a9f822b56
164 changed files with 82726 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

372
src/publish_assistant/generate_content.py
View File

@@ -127,19 +127,45 @@ def load_deadlines(path: str) -> dict[str, dict]:
return data.get("deadlines", {})
def load_venue_digests(papers_dir: Path) -> dict[str, list[dict]]:
"""Return {acronym: [digest_info_dict, ...]} sorted newest-first."""
result: dict[str, list[dict]] = {}
if not papers_dir.exists():
return result
for f in sorted(papers_dir.glob("*-digest.yaml"), reverse=True):
with open(f, encoding="utf-8") as fp:
d = yaml.safe_load(fp) or {}
venue = d.get("venue", "")
year = d.get("year")
if venue and year:
result.setdefault(venue, []).append({
"year": int(year),
"slug": f"{venue}-{year}",
"paper_count": len(d.get("selected") or d.get("papers") or []),
"tags": d.get("tags", []),
"date": str(d.get("date", f"{year}-01-01")),
})
return result
# ---------------------------------------------------------------------------
# Markdown body renderers
# ---------------------------------------------------------------------------
def _conf_body(fm: dict) -> str:
rank = fm.get("icore_rank") or "—"
hp = fm.get("homepage") or ""
dl = fm.get("next_deadline") or "TBD"
abstract_dl = fm.get("abstract_deadline") or ""
event = fm.get("next_event") or "—"
domains = ", ".join(f"`{d}`" for d in (fm.get("domain") or []))
notes = fm.get("notes") or ""
src = fm.get("deadline_source") or ""
def _conf_body(fm: dict, digests: list[dict] | None = None) -> str:
rank = fm.get("icore_rank") or "—"
hp = fm.get("homepage") or ""
dl = fm.get("next_deadline") or "TBD"
abstract_dl = fm.get("abstract_deadline") or ""
notification = fm.get("notification") or ""
camera_ready = fm.get("camera_ready") or ""
event = fm.get("next_event") or "—"
location = fm.get("location") or ""
cfp_url = fm.get("cfp_url") or ""
domains = ", ".join(f"`{d}`" for d in (fm.get("domain") or []))
notes = fm.get("notes") or ""
src = fm.get("deadline_source") or ""
title = fm.get("title", "")
lines = [f"\n*{fm.get('full_name', '')}*\n"]
lines += [
@@ -155,18 +181,34 @@ def _conf_body(fm: dict) -> str:
]
if abstract_dl and abstract_dl != dl:
lines.append(f"| **Abstract deadline** | {abstract_dl} |")
lines += [
f"| **Paper deadline** | {dl} |",
f"| **Event dates** | {event} |",
]
lines.append(f"| **Paper deadline** | {dl} |")
if notification:
lines.append(f"| **Notification** | {notification} |")
if camera_ready:
lines.append(f"| **Camera-ready** | {camera_ready} |")
lines.append(f"| **Event dates** | {event} |")
if location:
lines.append(f"| **Location** | {location} |")
if cfp_url:
lines.append(f"| **CFP** | [{cfp_url}]({cfp_url}) |")
if src:
lines.append(f"| **Source** | `{src}` |")
if digests:
lines += ["\n## Digests\n"]
for d in sorted(digests, key=lambda x: x["year"], reverse=True):
slug = d["slug"].lower()
year = d["year"]
count = d.get("paper_count", 0)
suffix = f"— {count} papers" if count else "*(digest pending)*"
lines.append(f"- [{title} {year}](/digests/{slug}/) {suffix}")
if notes:
lines += ["\n## Notes\n", notes]
return "\n".join(lines) + "\n"
def _journal_body(fm: dict) -> str:
def _journal_body(fm: dict, digests: list[dict] | None = None) -> str:
q = fm.get("scimago_quartile") or "—"
sjr = fm.get("scimago_sjr") or "—"
h = fm.get("scimago_h_index") or "—"
@@ -175,6 +217,7 @@ def _journal_body(fm: dict) -> str:
model = (fm.get("submission_model") or "rolling").title()
domains = ", ".join(f"`{d}`" for d in (fm.get("domain") or []))
notes = fm.get("notes") or ""
title = fm.get("title", "")
lines = [f"\n*{fm.get('full_name', '')}*\n"]
lines += [
@@ -188,6 +231,16 @@ def _journal_body(fm: dict) -> str:
]
if hp:
lines.append(f"| **Website** | [{hp}]({hp}) |")
if digests:
lines += ["\n## Digests\n"]
for d in sorted(digests, key=lambda x: x["year"], reverse=True):
slug = d["slug"].lower()
year = d["year"]
count = d.get("paper_count", 0)
suffix = f"— {count} papers" if count else "*(digest pending)*"
lines.append(f"- [{title} {year}](/digests/{slug}/) {suffix}")
if notes:
lines += ["\n## Notes\n", notes]
return "\n".join(lines) + "\n"
@@ -257,19 +310,26 @@ def _venues_body(venues: dict) -> str:
def _conferences_section_body(venues: dict, icore: dict, deadlines: dict) -> str:
rows = []
for conf in venues.get("conferences") or []:
acronym = conf.get("acronym", "")
rank = icore.get(acronym.upper(), "—") or "—"
dl = deadlines.get(acronym, {})
deadline = dl.get("submission_deadline") or dl.get("abstract_deadline") or "TBD"
domains = ", ".join(f"`{d}`" for d in conf.get("domain", []))
rows.append((_ICORE_ORDER.get(rank, 99), acronym, conf.get("full_name", ""), rank, deadline, domains))
acronym = conf.get("acronym", "")
rank = icore.get(acronym.upper(), "—") or "—"
dl = deadlines.get(acronym, {})
abstract = dl.get("abstract_deadline") or ""
paper = dl.get("submission_deadline") or dl.get("abstract_deadline") or "TBD"
notif = dl.get("notification") or ""
event = dl.get("event_dates") or "—"
location = dl.get("location") or ""
rows.append((_ICORE_ORDER.get(rank, 99), acronym, conf.get("full_name", ""),
rank, abstract, paper, notif, event, location))
rows.sort(key=lambda r: (r[0], r[1]))
lines = ["\n| Conference | Full name | ICORE | Next deadline | Domains |",
"|---|---|---|---|---|"]
for _, acronym, full_name, rank, deadline, domains in rows:
link = f"[{acronym}](/venues/conferences/{acronym.lower()}/)"
lines.append(f"| {link} | {full_name} | **{rank}** | {deadline} | {domains} |")
lines = [
"\n| Conference | Full name | ICORE | Abstract | Paper deadline | Notification | Event | Location |",
"|---|---|---|---|---|---|---|---|",
]
for _, acronym, full_name, rank, abstract, paper, notif, event, location in rows:
link = f"[{acronym}](/venues/conferences/{acronym.lower()}/)"
abs_cell = abstract if (abstract and abstract != paper) else ""
lines.append(f"| {link} | {full_name} | **{rank}** | {abs_cell} | {paper} | {notif} | {event} | {location} |")
return "\n".join(lines) + "\n"
@@ -277,18 +337,21 @@ def _journals_section_body(venues: dict) -> str:
rows = []
for j in venues.get("journals") or []:
acronym = j.get("acronym", "")
q = j.get("scimago_quartile") or "—"
sjr = j.get("scimago_sjr") or "—"
model = (j.get("submission_model") or "rolling").title()
q = j.get("scimago_quartile") or "—"
sjr = j.get("scimago_sjr") or "—"
h = j.get("scimago_h_index") or "—"
model = (j.get("submission_model") or "rolling").title()
domains = ", ".join(f"`{d}`" for d in j.get("domain", []))
rows.append((_QUARTILE_ORDER.get(q, 99), acronym, j.get("full_name", ""), q, sjr, model, domains))
rows.append((_QUARTILE_ORDER.get(q, 99), acronym, j.get("full_name", ""), q, sjr, h, model, domains))
rows.sort(key=lambda r: (r[0], r[1]))
lines = ["\n| Journal | Full name | SCImago | SJR | Model | Domains |",
"|---|---|---|---|---|---|"]
for _, acronym, full_name, q, sjr, model, domains in rows:
lines = [
"\n| Journal | Full name | SCImago | SJR | H-Index | Model | Domains |",
"|---|---|---|---|---|---|---|",
]
for _, acronym, full_name, q, sjr, h, model, domains in rows:
link = f"[{acronym}](/venues/journals/{acronym.lower()}/)"
lines.append(f"| {link} | {full_name} | **{q}** | {sjr} | {model} | {domains} |")
lines.append(f"| {link} | {full_name} | **{q}** | {sjr} | {h} | {model} | {domains} |")
return "\n".join(lines) + "\n"
@@ -299,61 +362,75 @@ def _journals_section_body(venues: dict) -> str:
def _calendar_events(venues: dict, deadlines: dict) -> list[dict]:
events: list[dict] = []
for conf in venues.get("conferences") or []:
acronym = conf.get("acronym", "")
dl = deadlines.get(acronym, {})
url = f"/venues/conferences/{acronym.lower()}/"
acronym = conf.get("acronym", "")
dl = deadlines.get(acronym, {})
url = f"/venues/conferences/{acronym.lower()}/"
location = dl.get("location") or ""
cfp_url = dl.get("cfp_url") or ""
for field, label, color in [
("abstract_deadline", "abstract deadline", "#f4a261"),
("submission_deadline", "paper deadline", "#e63946"),
("submission_deadline", "paper deadline", "#e63946"),
("notification", "notification", "#2a9d8f"),
]:
iso = _parse_date(dl.get(field) or "")
if iso:
events.append({"title": f"{acronym} — {label}", "start": iso,
"url": url, "color": color, "allDay": True})
ev: dict = {"title": f"{acronym} — {label}", "start": iso,
"url": url, "color": color, "allDay": True}
props: dict = {}
if location:
props["location"] = location
if cfp_url:
props["cfp_url"] = cfp_url
if props:
ev["extendedProps"] = props
events.append(ev)
start, end = _parse_date_range(dl.get("event_dates") or "")
if start:
ev: dict = {"title": acronym, "start": start, "url": url,
"color": "#457b9d", "allDay": True}
ev = {"title": acronym, "start": start, "url": url,
"color": "#457b9d", "allDay": True}
if end:
ev["end"] = end
if location:
ev["extendedProps"] = {"location": location}
events.append(ev)
return events
def _calendar_body(venues: dict, icore: dict, deadlines: dict) -> str:
from datetime import datetime
def parse_deadline(s: str):
for fmt in ("%b %d, %Y", "%b %d %Y"):
try:
return datetime.strptime(s.strip(), fmt)
except ValueError:
pass
return datetime.max # TBD / unparseable goes last
return datetime.max
rows = []
for conf in venues.get("conferences") or []:
acronym = conf.get("acronym", "")
rank = icore.get(acronym.upper(), "—") or "—"
dl_data = deadlines.get(acronym, {})
deadline = (
dl_data.get("submission_deadline")
or dl_data.get("abstract_deadline")
or "TBD"
)
event = dl_data.get("event_dates") or "—"
rows.append((acronym, deadline, event, rank))
acronym = conf.get("acronym", "")
rank = icore.get(acronym.upper(), "—") or "—"
dl_data = deadlines.get(acronym, {})
abstract = dl_data.get("abstract_deadline") or ""
paper = dl_data.get("submission_deadline") or dl_data.get("abstract_deadline") or "TBD"
notif = dl_data.get("notification") or ""
event = dl_data.get("event_dates") or "—"
location = dl_data.get("location") or ""
rows.append((acronym, abstract, paper, notif, event, location, rank))
rows.sort(key=lambda r: (parse_deadline(r[1]), r[0]))
rows.sort(key=lambda r: (parse_deadline(r[2]), r[0]))
lines = [
"\n| Conference | Submission deadline | Event dates | ICORE |",
"|------------|---------------------|-------------|-------|",
"\n| Conference | Abstract | Paper deadline | Notification | Event dates | Location | ICORE |",
"|---|---|---|---|---|---|---|",
]
for acronym, dl, event, rank in rows:
lines.append(f"| [{acronym}](/venues/conferences/{acronym.lower()}/) | {dl} | {event} | {rank} |")
for acronym, abstract, paper, notif, event, location, rank in rows:
abs_cell = abstract if (abstract and abstract != paper) else ""
lines.append(
f"| [{acronym}](/venues/conferences/{acronym.lower()}/) "
f"| {abs_cell} | {paper} | {notif} | {event} | {location} | {rank} |"
)
lines += [
"",
@@ -364,17 +441,19 @@ def _calendar_body(venues: dict, icore: dict, deadlines: dict) -> str:
def _digest_body(papers: list[dict]) -> str:
if not papers:
return "\nNo papers selected yet.\n"
lines = [f"\n{len(papers)} papers selected.\n"]
for p in papers:
title = p.get("title") or "Untitled"
authors = p.get("authors") or []
title = p.get("title") or "Untitled"
authors = p.get("authors") or []
author_str = (
", ".join(authors[:4]) + (" *et al.*" if len(authors) > 4 else "")
if authors else ""
)
tldr = p.get("tldr") or ""
why = p.get("why_notable") or ""
link = p.get("url") or (f"https://doi.org/{p['doi']}" if p.get("doi") else "")
tldr = p.get("tldr") or ""
why = p.get("why_notable") or ""
link = p.get("url") or (f"https://doi.org/{p['doi']}" if p.get("doi") else "")
lines += ["---", f"\n### {title}\n"]
if author_str:
@@ -392,7 +471,8 @@ def _digest_body(papers: list[dict]) -> str:
# Content generators
# ---------------------------------------------------------------------------
def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path) -> int:
def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path,
venue_digests: dict[str, list[dict]] | None = None) -> int:
count = 0
for conf in venues.get("conferences") or []:
acronym = conf.get("acronym", "").strip()
@@ -418,19 +498,29 @@ def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path) -> i
new_fm["next_deadline"] = dl["abstract_deadline"]
if dl.get("abstract_deadline") and dl.get("submission_deadline"):
new_fm["abstract_deadline"] = dl["abstract_deadline"]
if dl.get("notification"):
new_fm["notification"] = dl["notification"]
if dl.get("camera_ready"):
new_fm["camera_ready"] = dl["camera_ready"]
if dl.get("event_dates"):
new_fm["next_event"] = dl["event_dates"]
if dl.get("location"):
new_fm["location"] = dl["location"]
if dl.get("cfp_url"):
new_fm["cfp_url"] = dl["cfp_url"]
if dl.get("wikicfp_event_id"):
new_fm["deadline_source"] = f"wikicfp:{dl['wikicfp_event_id']}"
fm = merge_fm(existing_fm, new_fm, PRESERVED_VENUE_FIELDS)
write_hugo_file(path, fm, _conf_body(fm))
digests = (venue_digests or {}).get(acronym)
write_hugo_file(path, fm, _conf_body(fm, digests))
count += 1
print(f" {path.relative_to(root.parent)}", file=sys.stderr)
return count
def gen_journals(venues: dict, scimago: dict[str, dict], root: Path) -> int:
def gen_journals(venues: dict, scimago: dict[str, dict], root: Path,
venue_digests: dict[str, list[dict]] | None = None) -> int:
count = 0
for journal in venues.get("journals") or []:
acronym = journal.get("acronym", "").strip()
@@ -459,7 +549,8 @@ def gen_journals(venues: dict, scimago: dict[str, dict], root: Path) -> int:
**sjr_data,
}
fm = merge_fm(existing_fm, new_fm, PRESERVED_VENUE_FIELDS)
write_hugo_file(path, fm, _journal_body(fm))
digests = (venue_digests or {}).get(acronym)
write_hugo_file(path, fm, _journal_body(fm, digests))
count += 1
print(f" {path.relative_to(root.parent)}", file=sys.stderr)
return count
@@ -504,52 +595,101 @@ def gen_section_indexes(venues: dict, icore: dict, deadlines: dict, root: Path)
print(f" {p.relative_to(root.parent)}", file=sys.stderr)
def gen_digests(papers_dir: Path, root: Path) -> int:
if not papers_dir.exists():
return 0
def _upcoming_year(acronym: str, deadlines: dict) -> int:
"""Extract the year of the upcoming event from deadlines, falling back to current year."""
event_dates = (deadlines.get(acronym) or {}).get("event_dates") or ""
m = re.search(r'\b(20\d\d)\b', event_dates)
return int(m.group(1)) if m else datetime.now().year
def gen_digests(papers_dir: Path, root: Path,
venues: dict | None = None, deadlines: dict | None = None) -> int:
real_slugs: set[str] = set()
count = 0
for digest_file in sorted(papers_dir.glob("*-digest.yaml")):
with open(digest_file, encoding="utf-8") as f:
digest = yaml.safe_load(f) or {}
venue = digest.get("venue", "")
year = digest.get("year")
if not venue or not year:
print(f" Skipping {digest_file.name}: missing venue or year", file=sys.stderr)
continue
# Load candidate pool for full metadata (authors, DOI, URL, etc.)
candidates: dict[str, dict] = {}
candidate_file = papers_dir / f"{venue}-{year}-candidates.yaml"
if candidate_file.exists():
with open(candidate_file, encoding="utf-8") as f:
pool = yaml.safe_load(f) or {}
for p in pool.get("papers", []):
key = p.get("dblp_key") or p.get("openalex_id") or p.get("doi") or p.get("title", "")
candidates[key] = p
if papers_dir.exists():
for digest_file in sorted(papers_dir.glob("*-digest.yaml")):
with open(digest_file, encoding="utf-8") as f:
digest = yaml.safe_load(f) or {}
venue = digest.get("venue", "")
year = digest.get("year")
if not venue or not year:
print(f" Skipping {digest_file.name}: missing venue or year", file=sys.stderr)
continue
# Build enriched paper list
papers_out = []
for sel in digest.get("selected", []):
key = sel.get("dblp_key") or sel.get("openalex_id") or sel.get("doi") or sel.get("title", "")
papers_out.append({**candidates.get(key, {}), **sel})
candidates: dict[str, dict] = {}
candidate_file = papers_dir / f"{venue}-{year}-candidates.yaml"
if candidate_file.exists():
with open(candidate_file, encoding="utf-8") as f:
pool = yaml.safe_load(f) or {}
for p in pool.get("papers", []):
key = p.get("dblp_key") or p.get("openalex_id") or p.get("doi") or p.get("title", "")
candidates[key] = p
slug = f"{venue}-{year}"
path = root / "digests" / slug / "index.md"
existing_fm, _ = read_hugo_file(path)
items = digest.get("selected") or []
if not items:
for p in digest.get("papers") or []:
items.append({
"title": p.get("title", ""),
"authors": p.get("authors", []),
"tldr": p.get("reason", ""),
"why_notable": "",
})
papers_out = []
for sel in items:
key = sel.get("dblp_key") or sel.get("openalex_id") or sel.get("doi") or sel.get("title", "")
papers_out.append({**candidates.get(key, {}), **sel})
slug = f"{venue}-{year}"
real_slugs.add(slug)
path = root / "digests" / slug / "index.md"
existing_fm, _ = read_hugo_file(path)
new_fm = {
"title": f"{venue} {year} Digest",
"venue": venue,
"year": int(year),
"date": str(digest.get("date", f"{year}-01-01")),
"tags": digest.get("tags", []),
"paper_count": len(papers_out),
"draft": False,
}
fm = merge_fm(existing_fm, new_fm, set())
write_hugo_file(path, fm, _digest_body(papers_out))
count += 1
print(f" {path.relative_to(root.parent)}", file=sys.stderr)
# Generate stubs for every venue that has no digest for its upcoming edition
if venues and deadlines is not None:
all_acronyms = (
[c.get("acronym", "") for c in (venues.get("conferences") or [])] +
[j.get("acronym", "") for j in (venues.get("journals") or [])]
)
for acronym in all_acronyms:
if not acronym:
continue
year = _upcoming_year(acronym, deadlines)
slug = f"{acronym}-{year}"
if slug in real_slugs:
continue
path = root / "digests" / slug / "index.md"
if path.exists():
continue # never overwrite existing stubs (may have been customised)
new_fm = {
"title": f"{acronym} {year} Digest",
"venue": acronym,
"year": year,
"date": f"{year}-01-01",
"tags": [],
"paper_count": 0,
"draft": False,
"build": {"list": "never"},
}
write_hugo_file(path, new_fm, _digest_body([]))
count += 1
print(f" {path.relative_to(root.parent)} (stub)", file=sys.stderr)
new_fm = {
"title": f"{venue} {year} Digest",
"venue": venue,
"year": int(year),
"date": str(digest.get("date", f"{year}-01-01")),
"tags": digest.get("tags", []),
"paper_count": len(papers_out),
"draft": False,
}
fm = merge_fm(existing_fm, new_fm, set())
write_hugo_file(path, fm, _digest_body(papers_out))
count += 1
print(f" {path.relative_to(root.parent)}", file=sys.stderr)
return count
@@ -576,9 +716,11 @@ def main() -> None:
venues = yaml.safe_load(f) or {}
content_root = Path(args.content)
icore = load_icore(args.icore)
scimago = load_scimago(args.scimago)
deadlines = load_deadlines(args.deadlines)
papers_dir = Path(args.papers_dir)
icore = load_icore(args.icore)
scimago = load_scimago(args.scimago)
deadlines = load_deadlines(args.deadlines)
venue_digests = load_venue_digests(papers_dir)
if icore: print(f"Loaded {len(icore)} ICORE entries", file=sys.stderr)
if scimago: print(f"Loaded {len(scimago)} SCImago entries", file=sys.stderr)
@@ -588,16 +730,16 @@ def main() -> None:
gen_section_indexes(venues, icore, deadlines, content_root)
print("\nGenerating conference pages …", file=sys.stderr)
n_conf = gen_conferences(venues, icore, deadlines, content_root)
n_conf = gen_conferences(venues, icore, deadlines, content_root, venue_digests)
print("\nGenerating journal pages …", file=sys.stderr)
n_jour = gen_journals(venues, scimago, content_root)
n_jour = gen_journals(venues, scimago, content_root, venue_digests)
print("\nGenerating calendar page …", file=sys.stderr)
gen_calendar(venues, icore, deadlines, content_root)
print("\nGenerating digest pages …", file=sys.stderr)
n_dig = gen_digests(Path(args.papers_dir), content_root)
n_dig = gen_digests(papers_dir, content_root, venues, deadlines)
print(f"\nDone: {n_conf} conferences, {n_jour} journals, {n_dig} digests", file=sys.stderr)