feat(plugin-catalog): rank entries by GitHub stars, probed at most once a day
Catalog entries sort official → stars desc → name, both in browse shelves and
filtered grids, with a ★ pill on each card linking to the repo's stargazers.
Rate-limit discipline is the design constraint: the docs site deploys many
times a day and shares one GitHub App API budget with every other workflow
(tonight's merge train got rate-limited on unrelated uploads). So
website/scripts/fetch-plugin-stars.py first fetches the live site's own
plugin-stars.json (a CDN GET, not the API); if that cache is under 24h old it
is reused verbatim and GitHub is never called. Only a stale cache triggers one
GET /repos/{owner}/{repo} per unique catalog repo, and a 403/429 mid-run keeps
the previous counts instead of zeroing them. extract-plugins.py merges the
cache into plugins.json (`stars`) and plugins-meta.json (`starsFetchedAt`), and
the page footnote says when the ranking was last refreshed.
This commit is contained in:
@@ -142,6 +142,12 @@ def test_main_writes_catalog_and_meta(mod, tmp_path):
|
||||
catalog.mkdir()
|
||||
_write_entry(catalog, "alpha", tier="official", category="memory")
|
||||
_write_entry(catalog, "beta") # no category → default "desktop" shelf
|
||||
_write_entry(catalog, "gamma")
|
||||
# Star cache from fetch-plugin-stars.py: gamma outranks beta within the community tier.
|
||||
(tmp_path / "api").mkdir()
|
||||
(tmp_path / "api" / "plugin-stars.json").write_text(json.dumps({
|
||||
"fetched_at": "2026-09-15T00:00:00+00:00",
|
||||
"stars": {"example/gamma": 50, "example/beta": 3}}), encoding="utf-8")
|
||||
(catalog / "removed.yaml").write_text(
|
||||
"removed:\n - name: gone\n", encoding="utf-8"
|
||||
)
|
||||
@@ -152,17 +158,19 @@ def test_main_writes_catalog_and_meta(mod, tmp_path):
|
||||
assert rc == 0
|
||||
plugins = json.loads((out_dir / "plugins.json").read_text(encoding="utf-8"))
|
||||
meta = json.loads((out_dir / "plugins-meta.json").read_text(encoding="utf-8"))
|
||||
assert [p["name"] for p in plugins] == ["alpha", "beta"]
|
||||
assert meta["total"] == 2
|
||||
assert meta["byTier"] == {"official": 1, "community": 1}
|
||||
assert {p["name"]: p["category"] for p in plugins} == {"alpha": "memory", "beta": "desktop"}
|
||||
assert meta["byCategory"] == {"desktop": 1, "memory": 1}
|
||||
assert [p["name"] for p in plugins] == ["alpha", "gamma", "beta"] # official first, then stars desc
|
||||
assert {p["name"]: p["stars"] for p in plugins} == {"alpha": None, "gamma": 50, "beta": 3}
|
||||
assert meta["total"] == 3
|
||||
assert meta["byTier"] == {"official": 1, "community": 2}
|
||||
assert {p["name"]: p["category"] for p in plugins} == {"alpha": "memory", "beta": "desktop", "gamma": "desktop"}
|
||||
assert meta["byCategory"] == {"desktop": 2, "memory": 1}
|
||||
assert meta["starsFetchedAt"] == "2026-09-15T00:00:00+00:00"
|
||||
assert meta["removedCount"] == 1
|
||||
assert meta["generatedAt"]
|
||||
# The live-refresh document consumed by installed clients: loader-schema entries + the kill list.
|
||||
from hermes_cli.plugin_catalog import entry_from_mapping
|
||||
live = json.loads((out_dir / "plugin-catalog.json").read_text(encoding="utf-8"))
|
||||
assert [entry_from_mapping(raw, "live").name for raw in live["entries"]] == ["alpha", "beta"]
|
||||
assert [entry_from_mapping(raw, "live").name for raw in live["entries"]] == ["alpha", "beta", "gamma"]
|
||||
assert live["removed"] == [{"name": "gone"}]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""fetch-plugin-stars.py: the daily GitHub-stars cache behind catalog ranking.
|
||||
|
||||
The contract under test is rate-limit discipline, not the numbers: a fresh cache must never
|
||||
reach GitHub, and a rate-limited probe must keep the previous counts rather than zeroing them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import urllib.error
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "website" / "scripts" / "fetch-plugin-stars.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mod():
|
||||
spec = importlib.util.spec_from_file_location("fetch_plugin_stars", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _catalog(tmp_path: Path, *repos: str) -> Path:
|
||||
import yaml
|
||||
|
||||
cat = tmp_path / "plugin-catalog"
|
||||
cat.mkdir()
|
||||
for i, repo in enumerate(repos):
|
||||
(cat / f"p{i}.yaml").write_text(yaml.safe_dump({
|
||||
"name": f"p{i}", "repo": repo, "sha": "38fe0fb53eff98d477f807432e965429e665ca33",
|
||||
"description": "d", "maintainer": "m"}), encoding="utf-8")
|
||||
return cat
|
||||
|
||||
|
||||
def test_fresh_cache_is_reused_without_any_github_call(mod, tmp_path, monkeypatch):
|
||||
cat = _catalog(tmp_path, "https://github.com/a/one")
|
||||
out = tmp_path / "plugin-stars.json"
|
||||
recent = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat()
|
||||
out.write_text(json.dumps({"fetched_at": recent, "stars": {"a/one": 7}}), encoding="utf-8")
|
||||
|
||||
def boom(*a, **k):
|
||||
raise AssertionError("GitHub must not be called while the cache is fresh")
|
||||
monkeypatch.setattr(mod, "_http_json", boom)
|
||||
|
||||
assert mod.main(catalog_dir=cat, output=out, max_age_hours=24, live_url=None) == 0
|
||||
assert json.loads(out.read_text())["stars"] == {"a/one": 7}
|
||||
|
||||
|
||||
def test_stale_cache_probes_and_rate_limit_keeps_previous_counts(mod, tmp_path, monkeypatch):
|
||||
cat = _catalog(tmp_path, "https://github.com/a/one", "https://github.com/b/two", "https://gitlab.com/c/three")
|
||||
out = tmp_path / "plugin-stars.json"
|
||||
old = (datetime.now(timezone.utc) - timedelta(days=3)).isoformat()
|
||||
out.write_text(json.dumps({"fetched_at": old, "stars": {"a/one": 7, "b/two": 9}}), encoding="utf-8")
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def fake(url, headers, timeout=15.0):
|
||||
calls.append(url)
|
||||
if url.endswith("/repos/a/one"):
|
||||
return {"stargazers_count": 42}
|
||||
raise urllib.error.HTTPError(url, 403, "rate limited", hdrs=None, fp=None)
|
||||
monkeypatch.setattr(mod, "_http_json", fake)
|
||||
|
||||
assert mod.main(catalog_dir=cat, output=out, max_age_hours=24, live_url=None) == 0
|
||||
data = json.loads(out.read_text())
|
||||
# a/one refreshed; b/two kept its old count instead of dropping to 0; gitlab never probed.
|
||||
assert data["stars"] == {"a/one": 42, "b/two": 9}
|
||||
assert calls == ["https://api.github.com/repos/a/one", "https://api.github.com/repos/b/two"]
|
||||
assert data["fetched_at"] > old
|
||||
Reference in New Issue
Block a user