feat(plugin-catalog): rank entries by GitHub stars, probed at most once a day

Catalog entries sort official → stars desc → name, both in browse shelves and
filtered grids, with a ★ pill on each card linking to the repo's stargazers.

Rate-limit discipline is the design constraint: the docs site deploys many
times a day and shares one GitHub App API budget with every other workflow
(tonight's merge train got rate-limited on unrelated uploads). So
website/scripts/fetch-plugin-stars.py first fetches the live site's own
plugin-stars.json (a CDN GET, not the API); if that cache is under 24h old it
is reused verbatim and GitHub is never called. Only a stale cache triggers one
GET /repos/{owner}/{repo} per unique catalog repo, and a 403/429 mid-run keeps
the previous counts instead of zeroing them. extract-plugins.py merges the
cache into plugins.json (`stars`) and plugins-meta.json (`starsFetchedAt`), and
the page footnote says when the ranking was last refreshed.
This commit is contained in:
teknium1
2026-09-14 21:24:33 -07:00
parent 437116f949
commit 3e2e2c50eb
9 changed files with 361 additions and 11 deletions
+14 -6
View File
@@ -142,6 +142,12 @@ def test_main_writes_catalog_and_meta(mod, tmp_path):
catalog.mkdir()
_write_entry(catalog, "alpha", tier="official", category="memory")
_write_entry(catalog, "beta") # no category → default "desktop" shelf
_write_entry(catalog, "gamma")
# Star cache from fetch-plugin-stars.py: gamma outranks beta within the community tier.
(tmp_path / "api").mkdir()
(tmp_path / "api" / "plugin-stars.json").write_text(json.dumps({
"fetched_at": "2026-09-15T00:00:00+00:00",
"stars": {"example/gamma": 50, "example/beta": 3}}), encoding="utf-8")
(catalog / "removed.yaml").write_text(
"removed:\n - name: gone\n", encoding="utf-8"
)
@@ -152,17 +158,19 @@ def test_main_writes_catalog_and_meta(mod, tmp_path):
assert rc == 0
plugins = json.loads((out_dir / "plugins.json").read_text(encoding="utf-8"))
meta = json.loads((out_dir / "plugins-meta.json").read_text(encoding="utf-8"))
assert [p["name"] for p in plugins] == ["alpha", "beta"]
assert meta["total"] == 2
assert meta["byTier"] == {"official": 1, "community": 1}
assert {p["name"]: p["category"] for p in plugins} == {"alpha": "memory", "beta": "desktop"}
assert meta["byCategory"] == {"desktop": 1, "memory": 1}
assert [p["name"] for p in plugins] == ["alpha", "gamma", "beta"] # official first, then stars desc
assert {p["name"]: p["stars"] for p in plugins} == {"alpha": None, "gamma": 50, "beta": 3}
assert meta["total"] == 3
assert meta["byTier"] == {"official": 1, "community": 2}
assert {p["name"]: p["category"] for p in plugins} == {"alpha": "memory", "beta": "desktop", "gamma": "desktop"}
assert meta["byCategory"] == {"desktop": 2, "memory": 1}
assert meta["starsFetchedAt"] == "2026-09-15T00:00:00+00:00"
assert meta["removedCount"] == 1
assert meta["generatedAt"]
# The live-refresh document consumed by installed clients: loader-schema entries + the kill list.
from hermes_cli.plugin_catalog import entry_from_mapping
live = json.loads((out_dir / "plugin-catalog.json").read_text(encoding="utf-8"))
assert [entry_from_mapping(raw, "live").name for raw in live["entries"]] == ["alpha", "beta"]
assert [entry_from_mapping(raw, "live").name for raw in live["entries"]] == ["alpha", "beta", "gamma"]
assert live["removed"] == [{"name": "gone"}]
+75
View File
@@ -0,0 +1,75 @@
"""fetch-plugin-stars.py: the daily GitHub-stars cache behind catalog ranking.
The contract under test is rate-limit discipline, not the numbers: a fresh cache must never
reach GitHub, and a rate-limited probe must keep the previous counts rather than zeroing them.
"""
from __future__ import annotations
import importlib.util
import json
import urllib.error
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "website" / "scripts" / "fetch-plugin-stars.py"
@pytest.fixture(scope="module")
def mod():
spec = importlib.util.spec_from_file_location("fetch_plugin_stars", SCRIPT)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _catalog(tmp_path: Path, *repos: str) -> Path:
import yaml
cat = tmp_path / "plugin-catalog"
cat.mkdir()
for i, repo in enumerate(repos):
(cat / f"p{i}.yaml").write_text(yaml.safe_dump({
"name": f"p{i}", "repo": repo, "sha": "38fe0fb53eff98d477f807432e965429e665ca33",
"description": "d", "maintainer": "m"}), encoding="utf-8")
return cat
def test_fresh_cache_is_reused_without_any_github_call(mod, tmp_path, monkeypatch):
cat = _catalog(tmp_path, "https://github.com/a/one")
out = tmp_path / "plugin-stars.json"
recent = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat()
out.write_text(json.dumps({"fetched_at": recent, "stars": {"a/one": 7}}), encoding="utf-8")
def boom(*a, **k):
raise AssertionError("GitHub must not be called while the cache is fresh")
monkeypatch.setattr(mod, "_http_json", boom)
assert mod.main(catalog_dir=cat, output=out, max_age_hours=24, live_url=None) == 0
assert json.loads(out.read_text())["stars"] == {"a/one": 7}
def test_stale_cache_probes_and_rate_limit_keeps_previous_counts(mod, tmp_path, monkeypatch):
cat = _catalog(tmp_path, "https://github.com/a/one", "https://github.com/b/two", "https://gitlab.com/c/three")
out = tmp_path / "plugin-stars.json"
old = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat()
out.write_text(json.dumps({"fetched_at": old, "stars": {"a/one": 7, "b/two": 9}}), encoding="utf-8")
calls: list[str] = []
def fake(url, headers, timeout=15.0):
calls.append(url)
if url.endswith("/repos/a/one"):
return {"stargazers_count": 42}
raise urllib.error.HTTPError(url, 403, "rate limited", hdrs=None, fp=None)
monkeypatch.setattr(mod, "_http_json", fake)
assert mod.main(catalog_dir=cat, output=out, max_age_hours=24, live_url=None) == 0
data = json.loads(out.read_text())
# a/one refreshed; b/two kept its old count instead of dropping to 0; gitlab never probed.
assert data["stars"] == {"a/one": 42, "b/two": 9}
assert calls == ["https://api.github.com/repos/a/one", "https://api.github.com/repos/b/two"]
assert data["fetched_at"] > old