multi-topic, publish from gh-pages branch

src/publish_assistant/generate_content.py

@@ -6,18 +6,18 @@ Each page has a minimal front matter block (for Hugo taxonomy / metadata) plus
 a rendered Markdown body so generic themes display content without custom templates.
 
 Reads:
-  site/data/venues.yaml                    master venue list
-  site/data/rankings/icore.csv             ICORE conference ranks
-  site/data/rankings/scimago.csv           SCImago journal ranks (optional)
-  site/data/deadlines.yaml                 deadline data
-  site/data/papers/<V>-<Y>-candidates.yaml paper pools
-  site/data/papers/<V>-<Y>-digest.yaml     agent-curated selections
+  site/data/<topic>/venues.yaml                    master venue list
+  site/data/rankings/icore.csv                     ICORE conference ranks (shared)
+  site/data/rankings/scimago.csv                   SCImago journal ranks (shared, optional)
+  site/data/<topic>/deadlines.yaml                 deadline data
+  site/data/<topic>/papers/<V>-<Y>-candidates.yaml paper pools
+  site/data/<topic>/papers/<V>-<Y>-digest.yaml     agent-curated selections
 
 Writes:
-  site/content/venues/conferences/<acronym>/_index.md
-  site/content/venues/journals/<acronym>/_index.md
-  site/content/calendar/_index.md
-  site/content/digests/<VENUE>-<YEAR>/index.md
+  site/content/<topic>/venues/conferences/<acronym>/_index.md
+  site/content/<topic>/venues/journals/<acronym>/_index.md
+  site/content/<topic>/calendar/_index.md
+  site/content/<topic>/digests/<VENUE>-<YEAR>/index.md
 
 PRESERVED_VENUE_FIELDS are never overwritten once set (manual edits survive).
 """
@@ -152,7 +152,7 @@ def load_venue_digests(papers_dir: Path) -> dict[str, list[dict]]:
 # Markdown body renderers
 # ---------------------------------------------------------------------------
 
-def _conf_body(fm: dict, digests: list[dict] | None = None) -> str:
+def _conf_body(fm: dict, digests: list[dict] | None = None, base_path: str = "") -> str:
     rank         = fm.get("icore_rank") or "—"
     hp           = fm.get("homepage") or ""
     dl           = fm.get("next_deadline") or "TBD"
@@ -175,6 +175,13 @@ def _conf_body(fm: dict, digests: list[dict] | None = None) -> str:
     ]
     if hp:
         lines.append(f"| **Website** | [{hp}]({hp}) |")
+    if digests:
+        latest = sorted(digests, key=lambda x: x["year"], reverse=True)[0]
+        slug = latest["slug"].lower()
+        year = latest["year"]
+        count = latest.get("paper_count", 0)
+        suffix = f"— {count} papers" if count else "*(digest pending)*"
+        lines.append(f"| **Latest digest** | [{title} {year}]({base_path}/digests/{slug}/) {suffix} |")
     lines += [
         "\n## Upcoming Deadline\n",
         "| | |", "|---|---|",
@@ -195,20 +202,27 @@ def _conf_body(fm: dict, digests: list[dict] | None = None) -> str:
         lines.append(f"| **Source** | `{src}` |")
 
     if digests:
-        lines += ["\n## Digests\n"]
+        lines += ["\n## Previous Editions\n"]
         for d in sorted(digests, key=lambda x: x["year"], reverse=True):
             slug = d["slug"].lower()
             year = d["year"]
             count = d.get("paper_count", 0)
-            suffix = f"— {count} papers" if count else "*(digest pending)*"
-            lines.append(f"- [{title} {year}](/digests/{slug}/) {suffix}")
+            tags = d.get("tags") or []
+            lines.append(f"### {title} {year}\n")
+            lines += ["| | |", "|---|---|"]
+            if count:
+                lines.append(f"| **Papers digested** | {count} |")
+            if tags:
+                lines.append(f"| **Topics** | {', '.join(f'`{t}`' for t in tags)} |")
+            lines.append(f"| **Digest** | [→ {title} {year} Digest]({base_path}/digests/{slug}/) |")
+            lines.append("")
 
     if notes:
         lines += ["\n## Notes\n", notes]
     return "\n".join(lines) + "\n"
 
 
-def _journal_body(fm: dict, digests: list[dict] | None = None) -> str:
+def _journal_body(fm: dict, digests: list[dict] | None = None, base_path: str = "") -> str:
     q       = fm.get("scimago_quartile") or "—"
     sjr     = fm.get("scimago_sjr") or "—"
     h       = fm.get("scimago_h_index") or "—"
@@ -231,15 +245,29 @@ def _journal_body(fm: dict, digests: list[dict] | None = None) -> str:
     ]
     if hp:
         lines.append(f"| **Website** | [{hp}]({hp}) |")
+    if digests:
+        latest = sorted(digests, key=lambda x: x["year"], reverse=True)[0]
+        slug = latest["slug"].lower()
+        year = latest["year"]
+        count = latest.get("paper_count", 0)
+        suffix = f"— {count} papers" if count else "*(digest pending)*"
+        lines.append(f"| **Latest digest** | [{title} {year}]({base_path}/digests/{slug}/) {suffix} |")
 
     if digests:
-        lines += ["\n## Digests\n"]
+        lines += ["\n## Previous Editions\n"]
         for d in sorted(digests, key=lambda x: x["year"], reverse=True):
             slug = d["slug"].lower()
             year = d["year"]
             count = d.get("paper_count", 0)
-            suffix = f"— {count} papers" if count else "*(digest pending)*"
-            lines.append(f"- [{title} {year}](/digests/{slug}/) {suffix}")
+            tags = d.get("tags") or []
+            lines.append(f"### {title} {year}\n")
+            lines += ["| | |", "|---|---|"]
+            if count:
+                lines.append(f"| **Papers digested** | {count} |")
+            if tags:
+                lines.append(f"| **Topics** | {', '.join(f'`{t}`' for t in tags)} |")
+            lines.append(f"| **Digest** | [→ {title} {year} Digest]({base_path}/digests/{slug}/) |")
+            lines.append("")
 
     if notes:
         lines += ["\n## Notes\n", notes]
@@ -297,17 +325,18 @@ _ICORE_ORDER = {"A*": 0, "A": 1, "B": 2, "C": 3}
 _QUARTILE_ORDER = {"Q1": 0, "Q2": 1, "Q3": 2, "Q4": 3}
 
 
-def _venues_body(venues: dict) -> str:
+def _venues_body(venues: dict, base_path: str = "") -> str:
     n_conf = len(venues.get("conferences") or [])
     n_jour = len(venues.get("journals") or [])
     return (
         f"\nTracking **{n_conf} conferences** and **{n_jour} journals** for this domain.\n\n"
-        "- **[Conferences →](/venues/conferences/)** ranked by ICORE (A*, A, B, C)\n"
-        "- **[Journals →](/venues/journals/)** ranked by SCImago quartile (Q1–Q4)\n"
+        f"- **[Conferences →]({base_path}/venues/conferences/)** ranked by ICORE (A*, A, B, C)\n"
+        f"- **[Journals →]({base_path}/venues/journals/)** ranked by SCImago quartile (Q1–Q4)\n"
     )
 
 
-def _conferences_section_body(venues: dict, icore: dict, deadlines: dict) -> str:
+def _conferences_section_body(venues: dict, icore: dict, deadlines: dict,
+                              venue_digests: dict | None = None, base_path: str = "") -> str:
     rows = []
     for conf in venues.get("conferences") or []:
         acronym  = conf.get("acronym", "")
@@ -318,22 +347,25 @@ def _conferences_section_body(venues: dict, icore: dict, deadlines: dict) -> str
         notif    = dl.get("notification") or ""
         event    = dl.get("event_dates") or "—"
         location = dl.get("location") or ""
+        domains  = ", ".join(f"`{d}`" for d in conf.get("domain", []))
+        n_dig    = len((venue_digests or {}).get(acronym, []))
         rows.append((_ICORE_ORDER.get(rank, 99), acronym, conf.get("full_name", ""),
-                     rank, abstract, paper, notif, event, location))
+                     rank, abstract, paper, notif, event, location, domains, n_dig))
     rows.sort(key=lambda r: (r[0], r[1]))
 
     lines = [
-        "\n| Conference | Full name | ICORE | Abstract | Paper deadline | Notification | Event | Location |",
-        "|---|---|---|---|---|---|---|---|",
+        "\n| Conference | Full name | ICORE | Domains | Abstract | Paper deadline | Notification | Event | Location | Digests |",
+        "|---|---|---|---|---|---|---|---|---|---|",
     ]
-    for _, acronym, full_name, rank, abstract, paper, notif, event, location in rows:
-        link     = f"[{acronym}](/venues/conferences/{acronym.lower()}/)"
+    for _, acronym, full_name, rank, abstract, paper, notif, event, location, domains, n_dig in rows:
+        link     = f"[{acronym}]({base_path}/venues/conferences/{acronym.lower()}/)"
         abs_cell = abstract if (abstract and abstract != paper) else ""
-        lines.append(f"| {link} | {full_name} | **{rank}** | {abs_cell} | {paper} | {notif} | {event} | {location} |")
+        dig_cell = str(n_dig) if n_dig else "—"
+        lines.append(f"| {link} | {full_name} | **{rank}** | {domains} | {abs_cell} | {paper} | {notif} | {event} | {location} | {dig_cell} |")
     return "\n".join(lines) + "\n"
 
 
-def _journals_section_body(venues: dict) -> str:
+def _journals_section_body(venues: dict, venue_digests: dict | None = None, base_path: str = "") -> str:
     rows = []
     for j in venues.get("journals") or []:
         acronym = j.get("acronym", "")
@@ -342,16 +374,18 @@ def _journals_section_body(venues: dict) -> str:
         h       = j.get("scimago_h_index") or "—"
         model   = (j.get("submission_model") or "rolling").title()
         domains = ", ".join(f"`{d}`" for d in j.get("domain", []))
-        rows.append((_QUARTILE_ORDER.get(q, 99), acronym, j.get("full_name", ""), q, sjr, h, model, domains))
+        n_dig   = len((venue_digests or {}).get(acronym, []))
+        rows.append((_QUARTILE_ORDER.get(q, 99), acronym, j.get("full_name", ""), q, sjr, h, model, domains, n_dig))
     rows.sort(key=lambda r: (r[0], r[1]))
 
     lines = [
-        "\n| Journal | Full name | SCImago | SJR | H-Index | Model | Domains |",
-        "|---|---|---|---|---|---|---|",
+        "\n| Journal | Full name | SCImago | SJR | H-Index | Model | Domains | Digests |",
+        "|---|---|---|---|---|---|---|---|",
     ]
-    for _, acronym, full_name, q, sjr, h, model, domains in rows:
-        link = f"[{acronym}](/venues/journals/{acronym.lower()}/)"
-        lines.append(f"| {link} | {full_name} | **{q}** | {sjr} | {h} | {model} | {domains} |")
+    for _, acronym, full_name, q, sjr, h, model, domains, n_dig in rows:
+        link = f"[{acronym}]({base_path}/venues/journals/{acronym.lower()}/)"
+        dig_cell = str(n_dig) if n_dig else "—"
+        lines.append(f"| {link} | {full_name} | **{q}** | {sjr} | {h} | {model} | {domains} | {dig_cell} |")
     return "\n".join(lines) + "\n"
 
 
@@ -359,12 +393,12 @@ def _journals_section_body(venues: dict) -> str:
 # Calendar events builder (for FullCalendar)
 # ---------------------------------------------------------------------------
 
-def _calendar_events(venues: dict, deadlines: dict) -> list[dict]:
+def _calendar_events(venues: dict, deadlines: dict, base_path: str = "") -> list[dict]:
     events: list[dict] = []
     for conf in venues.get("conferences") or []:
         acronym  = conf.get("acronym", "")
         dl       = deadlines.get(acronym, {})
-        url      = f"/venues/conferences/{acronym.lower()}/"
+        url      = f"{base_path}/venues/conferences/{acronym.lower()}/"
         location = dl.get("location") or ""
         cfp_url  = dl.get("cfp_url") or ""
 
@@ -398,7 +432,7 @@ def _calendar_events(venues: dict, deadlines: dict) -> list[dict]:
     return events
 
 
-def _calendar_body(venues: dict, icore: dict, deadlines: dict) -> str:
+def _calendar_body(venues: dict, icore: dict, deadlines: dict, base_path: str = "") -> str:
     def parse_deadline(s: str):
         for fmt in ("%b %d, %Y", "%b %d %Y"):
             try:
@@ -428,7 +462,7 @@ def _calendar_body(venues: dict, icore: dict, deadlines: dict) -> str:
     for acronym, abstract, paper, notif, event, location, rank in rows:
         abs_cell = abstract if (abstract and abstract != paper) else ""
         lines.append(
-            f"| [{acronym}](/venues/conferences/{acronym.lower()}/) "
+            f"| [{acronym}]({base_path}/venues/conferences/{acronym.lower()}/) "
             f"| {abs_cell} | {paper} | {notif} | {event} | {location} | {rank} |"
         )
 
@@ -472,7 +506,8 @@ def _digest_body(papers: list[dict]) -> str:
 # ---------------------------------------------------------------------------
 
 def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path,
-                    venue_digests: dict[str, list[dict]] | None = None) -> int:
+                    venue_digests: dict[str, list[dict]] | None = None,
+                    base_path: str = "") -> int:
     count = 0
     for conf in venues.get("conferences") or []:
         acronym = conf.get("acronym", "").strip()
@@ -513,14 +548,15 @@ def gen_conferences(venues: dict, icore: dict, deadlines: dict, root: Path,
 
         fm = merge_fm(existing_fm, new_fm, PRESERVED_VENUE_FIELDS)
         digests = (venue_digests or {}).get(acronym)
-        write_hugo_file(path, fm, _conf_body(fm, digests))
+        write_hugo_file(path, fm, _conf_body(fm, digests, base_path))
         count += 1
         print(f"  {path.relative_to(root.parent)}", file=sys.stderr)
     return count
 
 
 def gen_journals(venues: dict, scimago: dict[str, dict], root: Path,
-                 venue_digests: dict[str, list[dict]] | None = None) -> int:
+                 venue_digests: dict[str, list[dict]] | None = None,
+                 base_path: str = "") -> int:
     count = 0
     for journal in venues.get("journals") or []:
         acronym = journal.get("acronym", "").strip()
@@ -550,32 +586,34 @@ def gen_journals(venues: dict, scimago: dict[str, dict], root: Path,
         }
         fm = merge_fm(existing_fm, new_fm, PRESERVED_VENUE_FIELDS)
         digests = (venue_digests or {}).get(acronym)
-        write_hugo_file(path, fm, _journal_body(fm, digests))
+        write_hugo_file(path, fm, _journal_body(fm, digests, base_path))
         count += 1
         print(f"  {path.relative_to(root.parent)}", file=sys.stderr)
     return count
 
 
-def gen_calendar(venues: dict, icore: dict, deadlines: dict, root: Path) -> None:
+def gen_calendar(venues: dict, icore: dict, deadlines: dict, root: Path,
+                 base_path: str = "") -> None:
     path = root / "calendar" / "_index.md"
     existing_fm, _ = read_hugo_file(path)
     new_fm = {
         "title": "Submission Deadlines",
         "layout": "calendar",
         "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-        "events": _calendar_events(venues, deadlines),
+        "events": _calendar_events(venues, deadlines, base_path),
         "draft": False,
     }
     fm = merge_fm(existing_fm, new_fm, set())
-    write_hugo_file(path, fm, _calendar_body(venues, icore, deadlines))
+    write_hugo_file(path, fm, _calendar_body(venues, icore, deadlines, base_path))
     print(f"  {path.relative_to(root.parent)}", file=sys.stderr)
 
 
-def gen_section_indexes(venues: dict, icore: dict, deadlines: dict, root: Path) -> None:
+def gen_section_indexes(venues: dict, icore: dict, deadlines: dict, root: Path,
+                        venue_digests: dict | None = None, base_path: str = "") -> None:
     # venues/_index.md
     p = root / "venues" / "_index.md"
     fm = merge_fm(read_hugo_file(p)[0], {"title": "Venues", "draft": False}, set())
-    write_hugo_file(p, fm, _venues_body(venues))
+    write_hugo_file(p, fm, _venues_body(venues, base_path))
     print(f"  {p.relative_to(root.parent)}", file=sys.stderr)
 
     # venues/conferences/_index.md
@@ -583,7 +621,7 @@ def gen_section_indexes(venues: dict, icore: dict, deadlines: dict, root: Path)
     n = len(venues.get("conferences") or [])
     fm = merge_fm(read_hugo_file(p)[0],
                   {"title": "Conferences", "description": f"{n} tracked conferences", "draft": False}, set())
-    write_hugo_file(p, fm, _conferences_section_body(venues, icore, deadlines))
+    write_hugo_file(p, fm, _conferences_section_body(venues, icore, deadlines, venue_digests, base_path))
     print(f"  {p.relative_to(root.parent)}", file=sys.stderr)
 
     # venues/journals/_index.md
@@ -591,7 +629,7 @@ def gen_section_indexes(venues: dict, icore: dict, deadlines: dict, root: Path)
     n = len(venues.get("journals") or [])
     fm = merge_fm(read_hugo_file(p)[0],
                   {"title": "Journals", "description": f"{n} tracked journals", "draft": False}, set())
-    write_hugo_file(p, fm, _journals_section_body(venues))
+    write_hugo_file(p, fm, _journals_section_body(venues, venue_digests, base_path))
     print(f"  {p.relative_to(root.parent)}", file=sys.stderr)
 
 
@@ -699,22 +737,25 @@ def gen_digests(papers_dir: Path, root: Path,
 
 def main() -> None:
     ap = argparse.ArgumentParser(description="Generate Hugo content from data files.")
-    ap.add_argument("--venues",      default="site/data/venues.yaml")
+    ap.add_argument("--venues",      default="site/data/cloud-edge/venues.yaml")
     ap.add_argument("--icore",       default="site/data/rankings/icore.csv")
     ap.add_argument("--scimago",     default="site/data/rankings/scimago.csv")
-    ap.add_argument("--deadlines",   default="site/data/deadlines.yaml")
-    ap.add_argument("--papers-dir",  default="site/data/papers", dest="papers_dir")
-    ap.add_argument("--content",     default="site/content")
+    ap.add_argument("--deadlines",   default="site/data/cloud-edge/deadlines.yaml")
+    ap.add_argument("--papers-dir",  default="site/data/cloud-edge/papers", dest="papers_dir")
+    ap.add_argument("--content",     default="site/content/cloud-edge")
+    ap.add_argument("--base-path",   default="/cloud-edge", dest="base_path",
+                    help="URL prefix for all internal links, e.g. '/cloud-edge'")
     args = ap.parse_args()
 
     venues_path = Path(args.venues)
     if not venues_path.exists():
-        print(f"Error: {venues_path} not found. Populate site/data/venues.yaml first.", file=sys.stderr)
+        print(f"Error: {venues_path} not found.", file=sys.stderr)
         sys.exit(1)
 
     with open(venues_path, encoding="utf-8") as f:
         venues = yaml.safe_load(f) or {}
 
+    base_path    = args.base_path.rstrip("/")
     content_root = Path(args.content)
     papers_dir   = Path(args.papers_dir)
     icore        = load_icore(args.icore)
@@ -727,16 +768,16 @@ def main() -> None:
     if deadlines: print(f"Loaded deadlines for {len(deadlines)} venues", file=sys.stderr)
 
     print("\nGenerating venue section indexes …", file=sys.stderr)
-    gen_section_indexes(venues, icore, deadlines, content_root)
+    gen_section_indexes(venues, icore, deadlines, content_root, venue_digests, base_path)
 
     print("\nGenerating conference pages …", file=sys.stderr)
-    n_conf = gen_conferences(venues, icore, deadlines, content_root, venue_digests)
+    n_conf = gen_conferences(venues, icore, deadlines, content_root, venue_digests, base_path)
 
     print("\nGenerating journal pages …", file=sys.stderr)
-    n_jour = gen_journals(venues, scimago, content_root, venue_digests)
+    n_jour = gen_journals(venues, scimago, content_root, venue_digests, base_path)
 
     print("\nGenerating calendar page …", file=sys.stderr)
-    gen_calendar(venues, icore, deadlines, content_root)
+    gen_calendar(venues, icore, deadlines, content_root, base_path)
 
     print("\nGenerating digest pages …", file=sys.stderr)
     n_dig = gen_digests(papers_dir, content_root, venues, deadlines)