+
+
+
+
+
+
diff --git a/src/builder.py b/src/builder.py
index 1c415d502a..3e1be981a5 100644
--- a/src/builder.py
+++ b/src/builder.py
@@ -10,11 +10,12 @@
from src import helpers
from src import pr_metrics
from src import azure_signing
+from src import catalog
from src.logger import log
def _safe_name(name: str) -> str:
- return os.path.basename(name)
+ return catalog.safe_key(name)
def _load_rtd_repos(base_dir: str) -> set:
@@ -27,8 +28,9 @@ def _load_rtd_repos(base_dir: str) -> set:
rtd_data = json.load(f)
for project in rtd_data:
git_url = project.get('repository', {}).get('url', '')
- repo_name = git_url.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
- rtd_repos.add(repo_name)
+ repo_name = catalog.github_name(git_url)
+ if repo_name:
+ rtd_repos.add(repo_name.lower())
except Exception:
pass
return rtd_repos
@@ -202,7 +204,7 @@ def _build_repo_entry(
rtd_repos: set,
code_scanning_open: int,
) -> dict:
- name = repo['name']
+ name = catalog.full_name(repo)
pr_count = len(prs)
if issues is None:
open_issues_total = repo.get('open_issues_count', 0)
@@ -224,6 +226,10 @@ def _build_repo_entry(
return {
'name': name,
+ 'id': repo.get('id'),
+ 'owner': (repo.get('owner') or {}).get('login'),
+ 'full_name': name,
+ 'html_url': repo.get('html_url'),
'stars': repo.get('stargazers_count', 0),
'forks': repo.get('forks_count', 0),
'issues': issue_count,
@@ -238,7 +244,7 @@ def _build_repo_entry(
'archived': repo.get('archived', False),
'fork': repo.get('fork', False),
'topics': repo.get('topics', []),
- 'has_readthedocs': name in rtd_repos,
+ 'has_readthedocs': any(alias.lower() in rtd_repos for alias in [name, *repo.get('dashboard_aliases', [])]),
'created_at': repo.get('created_at'),
'updated_at': repo.get('updated_at'),
}
@@ -272,21 +278,22 @@ def build():
if repo.get('private') or repo.get('archived'):
continue
- name = repo['name']
- coverage = _get_coverage(BASE_DIR, name)
- coverage_hist = _collect_coverage_history(BASE_DIR, name)
- coverage_history.extend(coverage_hist)
+ name = catalog.full_name(repo)
+ key = catalog.cache_key(repo)
+ coverage = _get_coverage(BASE_DIR, key)
+ coverage_hist = _collect_coverage_history(BASE_DIR, key)
+ coverage_history.extend({**entry, 'repo': name} for entry in coverage_hist)
if not coverage and coverage_hist:
coverage = max(coverage_hist, key=lambda e: e.get('date', '')).get('coverage', 0.0)
- languages = _get_languages(BASE_DIR, name)
- prs = _get_prs(BASE_DIR, name)
- issues = _get_issues(BASE_DIR, name)
- commit_activity.extend(_get_commit_activity(BASE_DIR, name))
- star_history.extend(_get_star_history(BASE_DIR, name))
- code_scanning_open = _get_code_scanning_open(BASE_DIR, name)
- code_scanning_history.extend(_get_code_scanning_history(BASE_DIR, name))
+ languages = _get_languages(BASE_DIR, key)
+ prs = _get_prs(BASE_DIR, key)
+ issues = _get_issues(BASE_DIR, key)
+ commit_activity.extend({**entry, 'repo': name} for entry in _get_commit_activity(BASE_DIR, key))
+ star_history.extend({**entry, 'repo': name} for entry in _get_star_history(BASE_DIR, key))
+ code_scanning_open = _get_code_scanning_open(BASE_DIR, key)
+ code_scanning_history.extend({**entry, 'repo': name} for entry in _get_code_scanning_history(BASE_DIR, key))
if pr_metrics.is_active_repo(repo):
- pr_metric_caches[name] = pr_metrics.load_cache(BASE_DIR, name)
+ pr_metric_caches[name] = pr_metrics.load_cache(BASE_DIR, key)
repos.append(_build_repo_entry(repo, coverage, languages, prs, issues, rtd_repos, code_scanning_open))
prs_all.extend({'repo': name, **pr} for pr in prs)
@@ -309,11 +316,16 @@ def write_json(filename, data):
write_json('pr_metrics.json', pr_metric_caches)
write_json('azure_signing.json', azure_signing.load_data(BASE_DIR))
now = datetime.now(timezone.utc)
+ collection = catalog.read_json(os.path.join(BASE_DIR, 'github', 'catalog.json'), {})
write_json('metadata.json', {
'updated_at': now.isoformat(),
'repo_count': len(repos),
+ 'organization_status': collection.get('organizations', []),
+ 'registry_fresh': collection.get('registry_fresh', True),
})
- pr_metrics.write_report_pages(TEMPLATE_DIR, pr_metric_caches, now)
+ aliases = {catalog.cache_key(repo): catalog.full_name(repo) for repo in raw_repos
+ if pr_metrics.is_active_repo(repo) and catalog.cache_key(repo) != catalog.full_name(repo)}
+ pr_metrics.write_report_pages(TEMPLATE_DIR, pr_metric_caches, now, aliases)
log.info('Dashboard build complete.')
diff --git a/src/catalog.py b/src/catalog.py
new file mode 100644
index 0000000000..25d8a4acd5
--- /dev/null
+++ b/src/catalog.py
@@ -0,0 +1,158 @@
+"""Organization discovery and persistent repository identities for public dashboard data."""
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+from src import helpers
+from src.logger import log
+
+ORGANIZATIONS_URL = 'https://raw.githubusercontent.com/LizardByte/.github/master/organizations.json'
+CATALOG_VERSION = 1
+
+
+def read_json(path: str, default):
+ """Read a restored cache without silently replacing malformed history."""
+ if not os.path.exists(path):
+ return default
+ with open(path, encoding='utf-8') as stream:
+ return json.load(stream)
+
+
+def safe_key(value: str) -> str:
+ """Accept a repository name or owner/name, never an arbitrary filesystem path."""
+ parts = value.split('/')
+ if len(parts) > 2 or any(not re.fullmatch(r'[\w.-]+', part) or part in {'.', '..'} for part in parts):
+ raise ValueError('Invalid repository cache key')
+ # Sanitize each filename component separately so owner/name keys stay distinct.
+ name = os.path.basename(parts[-1])
+ return os.path.join(os.path.basename(parts[0]), name) if len(parts) == 2 else name
+
+
+def full_name(repo: dict) -> str:
+ """Return the canonical GitHub identity, retaining support for legacy cached records."""
+ return repo.get('full_name') or repo['name']
+
+
+def github_name(url: str) -> str | None:
+ """Resolve HTTPS/SSH Read the Docs repository URLs without matching by basename."""
+ parsed = urlparse(url.replace('git@github.com:', 'https://github.com/'))
+ name = parsed.path.strip('/').removesuffix('.git')
+ if parsed.hostname != 'github.com' or name.count('/') != 1:
+ return None
+ safe_key(name)
+ return name
+
+
+def documentation_key(base_dir: str, url: str) -> str:
+ """Find the persistent storage identity for a docs project, including pre-transfer aliases."""
+ name = github_name(url)
+ if name is None:
+ raise ValueError('Read the Docs project has no GitHub repository identity')
+ repos = read_json(os.path.join(base_dir, 'github', 'repos.json'), [])
+ for repo in repos:
+ aliases = [full_name(repo), *repo.get('dashboard_aliases', [])]
+ if name.lower() in {alias.lower() for alias in aliases}:
+ return cache_key(repo)
+ return name
+
+
+def cache_key(repository) -> str:
+ """Return a stable storage key without relying on an owner's current repository name."""
+ if isinstance(repository, dict):
+ return repository.get('dashboard_key') or repository['name']
+ key = getattr(repository, 'dashboard_key', None)
+ return key if isinstance(key, str) else repository.name
+
+
+def load_state(base_dir: str) -> tuple[dict, list]:
+ """Restore the identity index, bootstrapping existing name-only URLs by repository ID."""
+ state = read_json(os.path.join(base_dir, 'github', 'catalog.json'),
+ {'version': CATALOG_VERSION, 'repositories': {}, 'organizations': []})
+ if state.get('version') != CATALOG_VERSION:
+ raise ValueError('Unsupported dashboard catalog version')
+ previous = read_json(os.path.join(base_dir, 'github', 'repos.json'), [])
+ for repo in previous:
+ if repo.get('private'):
+ continue
+ identity = str(repo['id'])
+ state['repositories'].setdefault(identity, {
+ 'key': repo.get('dashboard_key') or repo['name'], 'full_name': full_name(repo),
+ })
+ keys = [safe_key(record['key']).casefold() for record in state['repositories'].values()]
+ if len(keys) != len(set(keys)):
+ raise ValueError('Repository caches have conflicting identities')
+ return state, previous
+
+
+def organizations(state: dict) -> tuple[list[str], bool]:
+ """Fetch the shared registry, falling back only to a known prior registry on an outage."""
+ try:
+ response = helpers.s.get(url=ORGANIZATIONS_URL, timeout=30)
+ if response.status_code != 200:
+ raise ValueError('Organization registry request failed')
+ names = response.json()
+ if (not isinstance(names, list) or not names
+ or any(not isinstance(name, str) or not re.fullmatch(r'[a-z\d][a-z\d-]*', name, re.I)
+ for name in names)
+ or len({name.lower() for name in names}) != len(names)):
+ raise ValueError('Invalid organization registry')
+ return names, True
+ except Exception:
+ cached = [org['name'] for org in state['organizations']]
+ if not cached:
+ raise
+ log.warning('Organization registry unavailable; using the last successful registry.')
+ return cached, False
+
+
+def _register(state: dict, repo: dict) -> dict:
+ identity = str(repo['id'])
+ canonical = full_name(repo)
+ safe_key(canonical)
+ if identity not in state['repositories'] and any(
+ record['key'].lower() == canonical.lower() for record in state['repositories'].values()):
+ raise ValueError('Repository cache key belongs to another repository ID')
+ record = state['repositories'].setdefault(identity, {'key': canonical, 'full_name': canonical})
+ # Stable keys keep old public badge/banner paths valid across both transfers and renames.
+ aliases = sorted({*record.get('aliases', []), record['full_name'], canonical})
+ record.update({'full_name': canonical, 'aliases': aliases})
+ return {**repo, 'dashboard_key': record['key'], 'dashboard_aliases': aliases,
+ 'dashboard_paths': {
+ 'banner': f'github/openGraphImages/{record["key"]}.png',
+ 'banner_small': f'github/openGraphImages/{record["key"]}_624x312.png',
+ 'coverage_badge': f'shields/codecov/{record["key"]}.json',
+ }}
+
+
+def discover(github, base_dir: str) -> tuple[list, dict]:
+ """Collect each org independently, retaining failed-org snapshots and deduplicating by GitHub ID."""
+ state, previous = load_state(base_dir)
+ names, registry_fresh = organizations(state)
+ collected_at = datetime.now(timezone.utc).isoformat()
+ snapshots, fresh, statuses = {}, {}, []
+ for name in names:
+ try:
+ repos = list(github.get_organization(name).get_repos(type='public'))
+ public = {str(repo.raw_data['id']): repo for repo in repos if not repo.raw_data.get('private')}
+ fresh.update(public)
+ statuses.append({'name': name, 'status': 'ok', 'collected_at': collected_at})
+ except Exception:
+ cached = [repo for repo in previous if full_name(repo).split('/')[0].lower() == name.lower()
+ and not repo.get('private')]
+ snapshots.update((str(repo['id']), repo) for repo in cached)
+ prior = next((org for org in state['organizations'] if org['name'].lower() == name.lower()), {})
+ statuses.append({**prior, 'name': name, 'status': 'stale' if cached else 'unavailable'})
+ log.warning(f'GitHub organization {name} unavailable; preserving its cached public inventory.')
+ snapshots.update((identity, repo.raw_data) for identity, repo in fresh.items())
+ records = [_register(state, data) for data in snapshots.values()]
+ records.sort(key=lambda repo: full_name(repo).lower())
+ for identity, repo in fresh.items():
+ repo.dashboard_key = state['repositories'][identity]['key']
+ state.update({'source_url': ORGANIZATIONS_URL, 'registry_fresh': registry_fresh,
+ 'organizations': statuses})
+ helpers.write_json_files(os.path.join(base_dir, 'github', 'catalog'), state)
+ helpers.write_json_files(os.path.join(base_dir, 'github', 'repos'), records)
+ return list(fresh.values()), state
diff --git a/src/pr_metrics.py b/src/pr_metrics.py
index fad9680d4f..f860de4589 100644
--- a/src/pr_metrics.py
+++ b/src/pr_metrics.py
@@ -9,7 +9,7 @@
from urllib.parse import quote_plus
# local imports
-from src import helpers
+from src import helpers, catalog
GRAPHQL_URL = 'https://api.github.com/graphql'
GITHUB_OWNER = 'LizardByte'
@@ -119,7 +119,7 @@ def is_active_repo(repo: dict) -> bool:
def cache_path(base_dir: str, repository: str) -> str:
"""Return the cache path without its JSON extension."""
- return os.path.join(base_dir, 'github', 'prMetrics', os.path.basename(repository))
+ return os.path.join(base_dir, 'github', 'prMetrics', catalog.safe_key(repository))
def load_cache(base_dir: str, repository: str) -> dict | None:
@@ -246,7 +246,8 @@ def _fetch_connection(
if cutoff and updated_at and updated_at < cutoff:
reached_cutoff = True
break
- pulls.append(_normalize_pull(repository.name, pull))
+ name = catalog.full_name(getattr(repository, 'raw_data', {'name': repository.name}))
+ pulls.append(_normalize_pull(name, pull))
page_info = connection.get('pageInfo') or {}
if reached_cutoff or not page_info.get('hasNextPage'):
@@ -270,15 +271,16 @@ def fetch_repository(repository, headers: dict, session, now: datetime) -> list[
def refresh_repository(repository, base_dir: str, headers: dict, session, now: datetime | None = None) -> bool:
"""Refresh one repository cache when stale, returning whether it was written."""
now = now or datetime.now(tz=timezone.utc)
- existing = load_cache(base_dir, repository.name)
+ key = catalog.cache_key(repository)
+ existing = load_cache(base_dir, key)
if cache_is_fresh(existing, now):
return False
pulls = fetch_repository(repository, headers, session, now)
helpers.write_json_files(
- file_path=cache_path(base_dir, repository.name),
+ file_path=cache_path(base_dir, key),
data={
- 'repository': repository.name,
+ 'repository': catalog.full_name(getattr(repository, 'raw_data', {'name': repository.name})),
'collected_at': now.isoformat(),
'cache_version': CACHE_VERSION,
'history_days': HISTORY_DAYS,
@@ -432,14 +434,19 @@ def _markdown_text(value: object) -> str:
)
-def _github_search(repository: str | None, qualifiers: list[str]) -> str:
+def _github_search(repository: str | list[str] | None, qualifiers: list[str]) -> str:
"""Build a GitHub pull-request search URL for an organization or repository."""
- scope = f'repo:{GITHUB_OWNER}/{repository}' if repository else f'org:{GITHUB_OWNER}'
+ if isinstance(repository, list):
+ scope = ' '.join(f'org:{name}' for name in repository)
+ elif repository:
+ scope = f'repo:{repository}' if '/' in repository else f'repo:{GITHUB_OWNER}/{repository}'
+ else:
+ scope = f'org:{GITHUB_OWNER}'
query = ' '.join(['is:pr', scope, *qualifiers])
return f'https://github.com/pulls?q={quote_plus(query)}'
-def _metric_link(value: int, repository: str | None, *qualifiers: str) -> str:
+def _metric_link(value: int, repository: str | list[str] | None, *qualifiers: str) -> str:
"""Link a displayed metric count to the closest matching GitHub search."""
return f'[{value}]({_github_search(repository, list(qualifiers))})'
@@ -596,6 +603,8 @@ def render_repository_page(repository: str, cache: dict | None, now: datetime) -
def render_index_page(caches: dict[str, dict | None], now: datetime) -> str:
"""Render the organization-wide Jekyll Markdown report."""
+ organizations = sorted({name.split('/')[0] if '/' in name else GITHUB_OWNER for name in caches})
+ organizations = organizations or [GITHUB_OWNER]
all_pulls = [
pull
for cache in caches.values()
@@ -605,13 +614,13 @@ def render_index_page(caches: dict[str, dict | None], now: datetime) -> str:
report_cutoff = _search_timestamp(_window_cutoff(now, REPORT_DAYS))
stale_cutoff = _search_timestamp(_window_cutoff(now, STALE_DAYS))
overview_links = {
- 'open': _metric_link(totals['open'], None, SEARCH_OPEN),
- 'draft': _metric_link(totals['draft'], None, SEARCH_OPEN, SEARCH_DRAFT),
+ 'open': _metric_link(totals['open'], organizations, SEARCH_OPEN),
+ 'draft': _metric_link(totals['draft'], organizations, SEARCH_OPEN, SEARCH_DRAFT),
'not_reviewed': _metric_link(
- totals['not_reviewed'], None, SEARCH_OPEN, SEARCH_READY, SEARCH_NO_REVIEW),
+ totals['not_reviewed'], organizations, SEARCH_OPEN, SEARCH_READY, SEARCH_NO_REVIEW),
'approval': str(totals['awaiting_approval']),
- 'stale': _metric_link(totals['stale'], None, SEARCH_OPEN, f'updated:<{stale_cutoff}'),
- 'merged': _metric_link(totals['merged'], None, SEARCH_MERGED, f'merged:>={report_cutoff}'),
+ 'stale': _metric_link(totals['stale'], organizations, SEARCH_OPEN, f'updated:<{stale_cutoff}'),
+ 'merged': _metric_link(totals['merged'], organizations, SEARCH_MERGED, f'merged:>={report_cutoff}'),
}
lines = [
'---',
@@ -672,20 +681,25 @@ def render_index_page(caches: dict[str, dict | None], now: datetime) -> str:
return '\n'.join(lines)
-def write_report_pages(template_dir: str, caches: dict[str, dict | None], now: datetime) -> None:
+def write_report_pages(template_dir: str, caches: dict[str, dict | None], now: datetime,
+ aliases: dict[str, str] | None = None) -> None:
"""Write the organization index and all repository report pages."""
report_dir = Path(template_dir) / 'pr-metrics'
report_dir.mkdir(parents=True, exist_ok=True)
expected = {'index.md'}
- for repository, cache in caches.items():
- filename = f'{os.path.basename(repository)}.md'
+ pages = {**{name: name for name in caches}, **(aliases or {})}
+ for route, repository in pages.items():
+ filename = f'{catalog.safe_key(route)}.md'
expected.add(filename)
- (report_dir / filename).write_text(
- render_repository_page(repository, cache, now),
+ destination = report_dir / filename
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(
+ render_repository_page(repository, caches[repository], now).replace(
+ f'permalink: /pr-metrics/{repository}/', f'permalink: /pr-metrics/{route}/'),
encoding='utf-8',
)
(report_dir / 'index.md').write_text(render_index_page(caches, now), encoding='utf-8')
- for existing in report_dir.glob('*.md'):
- if existing.name not in expected:
+ for existing in report_dir.rglob('*.md'):
+ if str(existing.relative_to(report_dir)) not in expected:
existing.unlink()
diff --git a/src/updater.py b/src/updater.py
index 0a75492f0f..cbd7b0eda6 100644
--- a/src/updater.py
+++ b/src/updater.py
@@ -5,6 +5,7 @@
from queue import Queue
from datetime import datetime, timedelta, timezone
from threading import Thread
+from urllib.parse import urlparse
# lib imports
from github import Auth, Github
@@ -18,6 +19,7 @@
from src import helpers
from src import pr_metrics
from src import azure_signing
+from src import catalog
from src.logger import log
COMMIT_ACTIVITY_READY = 'ready'
@@ -119,56 +121,80 @@ def _build_codecov_shields_badge(data: dict) -> dict:
}
-def update_codecov():
- """
- Get code coverage data from Codecov API.
- """
+def _codecov_repositories(base_url: str, headers: dict):
+ """Paginate one org's public Codecov inventory without trusting cross-service next links."""
+ url = f'{base_url}/repos?page_size=500'
+ while url:
+ response = helpers.s.get(url=url, headers=headers)
+ try:
+ data = response.json()
+ except requests.exceptions.JSONDecodeError as error:
+ raise requests.exceptions.HTTPError('Invalid Codecov repository response') from error
+ if response.status_code != 200:
+ raise requests.exceptions.HTTPError(f'Codecov repository request failed (HTTP {response.status_code})')
+ yield from data['results']
+ url = data.get('next')
+ if url:
+ normalized = url.replace('https://api.codecov.io/', 'https://codecov.io/')
+ endpoint = urlparse(normalized)._replace(query='', fragment='').geturl().rstrip('/')
+ if endpoint != f'{base_url}/repos':
+ raise ValueError('Invalid Codecov pagination URL')
+
+
+def _merge_coverage_history(path: str, incoming: list) -> list:
+ """Keep historical source-org coverage when a destination Codecov project starts a new series."""
+ previous = catalog.read_json(f'{path}.json', [])
+ points = {entry['timestamp']: entry for entry in [*previous, *incoming] if entry.get('timestamp')}
+ return sorted(points.values(), key=lambda entry: entry['timestamp'])
+
+
+def _archived_codecov_repositories(owner: str, keys: dict | None) -> set:
+ """Get archived names in the current org without blocking collection on a missing cache."""
archived_repos = set()
repos_path = os.path.join(BASE_DIR, 'github', 'repos.json')
if os.path.exists(repos_path):
try:
with open(repos_path) as f:
- archived_repos = {r['name'] for r in json.load(f) if r.get('archived')}
+ archived_repos = {r['name'] for r in json.load(f) if r.get('archived')
+ and (keys is None or catalog.full_name(r).split('/')[0] == owner)}
except Exception as e:
log.warning(f'Could not load GitHub repos for archived check: {e}')
+ return archived_repos
+
+
+def _update_codecov_org(owner: str, keys: dict | None = None):
+ """Refresh one organization, retaining legacy name-only cache identities where assigned."""
+ archived_repos = _archived_codecov_repositories(owner, keys)
headers = {
'Accept': 'application/json',
'Authorization': f'bearer {os.environ["CODECOV_TOKEN"]}',
}
- base_url = f'https://codecov.io/api/v2/github/{os.environ["GITHUB_REPOSITORY_OWNER"]}'
-
- url = f'{base_url}/repos?page_size=500'
-
- response = helpers.s.get(url=url, headers=headers)
- try:
- data = response.json()
- except requests.exceptions.JSONDecodeError:
- log.error(f'Error: update_codecov: {response.text}')
- raise requests.exceptions.HTTPError(f'Error: {response.text}')
-
- if response.status_code != 200:
- log.error(f'Error: update_codecov: {data["detail"]}')
- raise requests.exceptions.HTTPError(f'Error: {data["detail"]}')
-
- assert data['next'] is None, 'More than 500 repos found, need to implement pagination.'
+ base_url = f'https://codecov.io/api/v2/github/{owner}'
+ fresh = True
for repo in tqdm(
- iterable=data['results'],
- desc='Updating Codecov data',
+ iterable=_codecov_repositories(base_url, headers),
+ desc=f'Updating Codecov data ({owner})',
):
- if repo['name'] in archived_repos:
+ if (repo.get('active') is False or repo['name'] in archived_repos
+ or (keys is not None and repo['name'] not in keys)):
continue
+ key = catalog.cache_key(keys[repo['name']]) if keys is not None else repo['name']
# Get repo details
url = f'{base_url}/repos/{repo["name"]}'
response = helpers.s.get(url=url, headers=headers)
data = response.json()
+ if response.status_code != 200:
+ log.warning(f'Codecov data unavailable for {owner}/{repo["name"]}; retaining cached data.')
+ fresh = False
+ continue
- file_path = os.path.join(BASE_DIR, 'codecov', repo['name'])
+ file_path = os.path.join(BASE_DIR, 'codecov', key)
helpers.write_json_files(file_path=file_path, data=data)
- badge_path = os.path.join(BASE_DIR, 'shields', 'codecov', repo['name'])
+ badge_path = os.path.join(BASE_DIR, 'shields', 'codecov', key)
helpers.write_json_files(file_path=badge_path, data=_build_codecov_shields_badge(data))
# Get coverage trend data
@@ -176,8 +202,28 @@ def update_codecov():
# Save coverage trend data
if coverage_trend_data:
- coverage_trend_path = os.path.join(BASE_DIR, 'codecov', f'{repo["name"]}_coverage_trend')
- helpers.write_json_files(file_path=coverage_trend_path, data=coverage_trend_data)
+ coverage_trend_path = os.path.join(BASE_DIR, 'codecov', f'{key}_coverage_trend')
+ helpers.write_json_files(file_path=coverage_trend_path,
+ data=_merge_coverage_history(coverage_trend_path, coverage_trend_data))
+ return fresh
+
+
+def update_codecov():
+ """Collect coverage separately for each discovered org, preserving failed-org caches."""
+ state = catalog.read_json(os.path.join(BASE_DIR, 'github', 'catalog.json'), None)
+ if state is None:
+ return _update_codecov_org(os.environ['GITHUB_REPOSITORY_OWNER'])
+ records = catalog.read_json(os.path.join(BASE_DIR, 'github', 'repos.json'), [])
+ for org in state['organizations']:
+ keys = {repo['name']: repo for repo in records
+ if catalog.full_name(repo).split('/')[0].lower() == org['name'].lower() and not repo.get('private')}
+ try:
+ fresh = _update_codecov_org(org['name'], keys)
+ org['codecov_status'] = 'ok' if fresh else 'stale'
+ except Exception:
+ org['codecov_status'] = 'stale'
+ log.warning(f'Codecov organization {org["name"]} unavailable; retaining its cached data.')
+ helpers.write_json_files(os.path.join(BASE_DIR, 'github', 'catalog'), state)
def update_discord():
@@ -265,7 +311,7 @@ def _commit_activity_cache_path(repo) -> str:
str
File path without the ``.json`` extension.
"""
- return os.path.join(BASE_DIR, 'github', 'commitActivity', repo.name)
+ return os.path.join(BASE_DIR, 'github', 'commitActivity', catalog.cache_key(repo))
def _commit_activity_hash_cache_path(repo) -> str:
@@ -282,7 +328,7 @@ def _commit_activity_hash_cache_path(repo) -> str:
str
File path without the ``.json`` extension.
"""
- return os.path.join(BASE_DIR, 'github', 'commitActivityHashes', repo.name)
+ return os.path.join(BASE_DIR, 'github', 'commitActivityHashes', catalog.cache_key(repo))
def _has_cached_commit_activity(repo) -> bool:
@@ -585,7 +631,7 @@ def _collect_star_history(repo, initial_samples: int = 5) -> list:
return []
today = datetime.now(tz=timezone.utc).strftime('%Y-%m-%d')
- cache_path = os.path.join(BASE_DIR, 'github', 'starHistory', f'{repo.name}.json')
+ cache_path = os.path.join(BASE_DIR, 'github', 'starHistory', f'{catalog.cache_key(repo)}.json')
existing = []
if os.path.exists(cache_path):
@@ -608,7 +654,7 @@ def _collect_star_history(repo, initial_samples: int = 5) -> list:
return history
-def _fetch_code_scanning_alerts(repo) -> list:
+def _fetch_code_scanning_alerts(repo) -> list | None:
"""
Fetch all code scanning alerts for a repository.
@@ -619,8 +665,8 @@ def _fetch_code_scanning_alerts(repo) -> list:
Returns
-------
- list
- List of alert objects, or an empty list when unavailable.
+ list | None
+ Alert objects, or None when analysis is unavailable; retain existing caches in that case.
"""
try:
alerts = list(repo.get_codescan_alerts())
@@ -628,7 +674,8 @@ def _fetch_code_scanning_alerts(repo) -> list:
except GithubException as e:
if e.status == 404:
log.warning(f'No code scanning analysis for {repo.name}, skipping code scanning alerts.')
- return []
+ return None
+ raise
def _alert_dates(alert) -> set:
@@ -911,19 +958,19 @@ def _process_github_repo(repo, headers: dict, graphql_url: str) -> None:
# languages
languages = _run_github_repo_step(repo, 'languages', repo.get_languages)
if languages is not None:
- file_path = os.path.join(BASE_DIR, 'github', 'languages', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'languages', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data=languages)
# open pull requests
pulls_data = _run_github_repo_step(repo, 'pull requests', lambda: _collect_open_pulls(repo))
if pulls_data is not None:
- file_path = os.path.join(BASE_DIR, 'github', 'pulls', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'pulls', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data=pulls_data)
# open issues
issues_data = _run_github_repo_step(repo, 'issues', lambda: _collect_open_issues(repo))
if issues_data is not None:
- file_path = os.path.join(BASE_DIR, 'github', 'issues', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'issues', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data=issues_data)
# open code scanning alerts and per-day history
@@ -932,7 +979,7 @@ def _process_github_repo(repo, headers: dict, graphql_url: str) -> None:
open_alert_count = sum(
1 for a in alerts if getattr(a, 'state', None) == 'open'
)
- file_path = os.path.join(BASE_DIR, 'github', 'codeScanning', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'codeScanning', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data={
'repo': repo.name,
'open': open_alert_count,
@@ -940,13 +987,13 @@ def _process_github_repo(repo, headers: dict, graphql_url: str) -> None:
})
code_scanning_history = _build_code_scanning_history(alerts)
- file_path = os.path.join(BASE_DIR, 'github', 'codeScanningHistory', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'codeScanningHistory', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data=code_scanning_history)
# star history (sampled to cap API calls)
star_history = _run_github_repo_step(repo, 'star history', lambda: _collect_star_history(repo))
if star_history:
- file_path = os.path.join(BASE_DIR, 'github', 'starHistory', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'starHistory', catalog.cache_key(repo))
helpers.write_json_files(file_path=file_path, data=star_history)
# openGraphImages - uses GraphQL
@@ -956,7 +1003,7 @@ def _process_github_repo(repo, headers: dict, graphql_url: str) -> None:
lambda: _fetch_open_graph_image_url(repo, headers, graphql_url),
)
if image_url and 'avatars' not in image_url:
- file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', repo.name)
+ file_path = os.path.join(BASE_DIR, 'github', 'openGraphImages', catalog.cache_key(repo))
_run_github_repo_step(
repo,
'OpenGraph image download',
@@ -978,19 +1025,7 @@ def update_github():
g = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"]), timeout=30)
g.per_page = 100
- # Get the user/organization
- owner = g.get_user(os.environ["GITHUB_REPOSITORY_OWNER"])
-
- # Get all repositories
- repos = list(owner.get_repos())
-
- # Convert PyGithub repo objects to dict format for JSON serialization
- repos_data = []
- for repo in repos:
- repos_data.append(repo.raw_data)
-
- file_path = os.path.join(BASE_DIR, 'github', 'repos')
- helpers.write_json_files(file_path=file_path, data=repos_data)
+ repos, _ = catalog.discover(g, BASE_DIR)
# GraphQL query still uses direct requests
headers = {
@@ -1084,7 +1119,7 @@ def update_readthedocs():
desc='Updating Readthedocs data',
):
git_url = project['repository']['url']
- repo_name = git_url.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
+ repo_name = catalog.documentation_key(BASE_DIR, git_url)
skip_links = [
'builds', # skip builds, too much data and too slow
@@ -1142,15 +1177,8 @@ def update():
target=update_patreon,
threads=independent_threads,
)
- append_thread_if_env_set(
- env_vars=['READTHEDOCS_TOKEN'],
- name='readthedocs',
- target=update_readthedocs,
- threads=independent_threads,
- )
- # GitHub must finish before Codecov so that repos.json is up to date and
- # update_codecov() can correctly skip archived repos.
+ # Dependent collectors need current repository identities and archive flags.
github_threads = []
append_thread_if_env_set(
env_vars=['GITHUB_TOKEN', 'GITHUB_REPOSITORY_OWNER'],
@@ -1159,12 +1187,18 @@ def update():
threads=github_threads,
)
- codecov_threads = []
+ repository_threads = []
append_thread_if_env_set(
env_vars=['CODECOV_TOKEN', 'GITHUB_REPOSITORY_OWNER'],
name='codecov',
target=update_codecov,
- threads=codecov_threads,
+ threads=repository_threads,
+ )
+ append_thread_if_env_set(
+ env_vars=['READTHEDOCS_TOKEN'],
+ name='readthedocs',
+ target=update_readthedocs,
+ threads=repository_threads,
)
# setup threading exception handling
@@ -1175,16 +1209,16 @@ def update():
for thread in tqdm(iterable=independent_threads + github_threads, desc='Starting threads'):
thread.start()
- # Wait for GitHub before starting Codecov.
+ # Wait for GitHub identities before starting Codecov and Read the Docs.
for thread in tqdm(iterable=github_threads, desc='Waiting for GitHub thread'):
thread.join()
- # Phase 2: start Codecov now that repos.json is fresh.
- for thread in tqdm(iterable=codecov_threads, desc='Starting Codecov thread'):
+ # Phase 2: collect services using the refreshed repository identities.
+ for thread in tqdm(iterable=repository_threads, desc='Starting repository service threads'):
thread.start()
# Wait for all remaining threads.
- for thread in tqdm(iterable=independent_threads + codecov_threads, desc='Waiting for threads to finish'):
+ for thread in tqdm(iterable=independent_threads + repository_threads, desc='Waiting for threads to finish'):
thread.join()
# deactivate threading exception handling
diff --git a/tests/dashboard.test.js b/tests/dashboard.test.js
index a2642044fa..ba2f0dbf67 100644
--- a/tests/dashboard.test.js
+++ b/tests/dashboard.test.js
@@ -82,7 +82,7 @@ describe('dashboard.js', () => {
buildDom();
globalThis.DASHBOARD_CONFIG = { base_path: '/dashboard' };
globalThis.matchMedia = jest.fn(() => ({ matches: false }));
- globalThis.Plotly = { newPlot: jest.fn() };
+ globalThis.Plotly = { newPlot: jest.fn(), purge: jest.fn() };
globalThis.DataTable = undefined;
globalThis.getComputedStyle = jest.fn(() => ({ getPropertyValue: () => '#fff' }));
globalThis.fetch = jest.fn(async () => ({ ok: true, json: async () => ({}) }));
@@ -151,6 +151,78 @@ describe('dashboard.js', () => {
expect(globalThis.Plotly.newPlot).toHaveBeenCalledTimes(1);
});
+ test('repository labels omit unique owners and disambiguate duplicate names on a separate line', () => {
+ const labels = mod.repositoryLabels([
+ 'LizardByte-infrastructure/upptime', 'First/demo', 'Second/demo', 'First/demo', 'legacy',
+ ]);
+ expect([...labels.values()]).toEqual(['upptime', 'demo
First', 'demo
Second', 'legacy']);
+ expect(mod.repositoryLabels(['First/demo']).get('First/demo')).toBe('demo');
+ });
+
+ test('category charts use readable ticks while keeping full identities and values for hover', () => {
+ const repos = ['First/demo', 'Second/demo', 'LizardByte-infrastructure/upptime'].map((name, index) => ({
+ name, value: index + 1, issues: index + 1, coverage: 90 + index,
+ }));
+ mod.renderBarChart('chart-stars', repos);
+ mod.renderIssuesBarChart(repos);
+ mod.renderPRsBarChart(repos, repos.map(repo => ({ repo: repo.name, draft: false })));
+ mod.renderCoverageChart(repos);
+ for (const id of ['chart-stars', 'chart-issues', 'chart-prs', 'chart-coverage']) {
+ const [, traces, layout] = globalThis.Plotly.newPlot.mock.calls.find(([chart]) => chart === id);
+ const ticks = new Map(layout.xaxis.tickvals.map((name, index) => [name, layout.xaxis.ticktext[index]]));
+ expect(ticks.get('LizardByte-infrastructure/upptime')).toBe('upptime');
+ expect(ticks.get('First/demo')).toBe('demo
First');
+ expect(ticks.get('Second/demo')).toBe('demo
Second');
+ expect(layout.xaxis.automargin).toBe(true);
+ expect(traces[0].x).toContain('First/demo');
+ expect(traces[0].x).toContain('Second/demo');
+ if (id === 'chart-coverage') {
+ expect(traces[0].text).toContain('First/demo: 90.0%');
+ } else {
+ expect(traces[0].customdata).toEqual(traces[0].x);
+ expect(traces[0].hovertemplate).toContain('%{customdata}');
+ }
+ }
+ });
+
+ test('history legends use short names without merging same-name series or hiding hover owners', () => {
+ const names = ['First/demo', 'Second/demo', 'LizardByte-infrastructure/upptime'];
+ const points = names.map((repo, index) => ({
+ repo, date: '2026-11-07', week: '2026-11-07', stars: index + 1, coverage: 90 + index,
+ open: index + 1, total: index + 1,
+ }));
+ mod.renderStarHistory(points);
+ mod.renderCoverageHistory(points);
+ mod.renderCodeScanningHistory(points);
+ mod.renderCommitActivityChart(points, names.map(name => ({ name })));
+ for (const id of ['chart-star-history', 'chart-coverage-history', 'chart-code-scanning-history',
+ 'chart-commit-activity-repos']) {
+ const [, traces] = globalThis.Plotly.newPlot.mock.calls.find(([chart]) => chart === id);
+ expect(traces).toHaveLength(3);
+ expect(traces.find(trace => trace.meta === 'First/demo').name).toBe('demo
First');
+ expect(traces.find(trace => trace.meta === 'Second/demo').name).toBe('demo
Second');
+ expect(traces.find(trace => trace.meta === 'LizardByte-infrastructure/upptime').name).toBe('upptime');
+ expect(traces.every(trace => trace.hovertemplate.includes('%{meta}'))).toBe(true);
+ }
+ });
+
+ test('treemap labels are short while hover data and node identities retain the owners', () => {
+ const repos = ['First/demo', 'Second/demo', 'LizardByte-infrastructure/upptime'].map(name => ({
+ name, languages: { Python: 10, JavaScript: 5 }, has_readthedocs: true, license: 'MIT',
+ }));
+ mod.renderLicenseChart(repos);
+ mod.renderDocsChart(repos);
+ mod.renderLanguageCharts(repos);
+ for (const [, [trace]] of globalThis.Plotly.newPlot.mock.calls) {
+ expect(trace.labels[trace.customdata.indexOf('LizardByte-infrastructure/upptime')]).toBe('upptime');
+ expect(trace.labels[trace.customdata.indexOf('First/demo')]).toBe('demo
First');
+ expect(trace.labels[trace.customdata.indexOf('Second/demo')]).toBe('demo
Second');
+ expect(trace.ids.some(id => id.includes('First/demo'))).toBe(true);
+ expect(trace.ids.some(id => id.includes('Second/demo'))).toBe(true);
+ expect(trace.hovertemplate).toContain('%{customdata}');
+ }
+ });
+
test('renderIssuesBarChart splits bot and other issues', () => {
const el = document.getElementById('chart-issues');
el.remove();
@@ -243,7 +315,8 @@ describe('dashboard.js', () => {
];
const draw = jest.fn();
- const dtCtor = jest.fn(() => ({ draw }));
+ const destroy = jest.fn();
+ const dtCtor = jest.fn(() => ({ draw, destroy }));
dtCtor.ext = { search: [] };
globalThis.DataTable = dtCtor;
@@ -264,6 +337,8 @@ describe('dashboard.js', () => {
globalThis.DataTable = undefined;
mod.renderPRTable(prs);
+ expect(destroy).toHaveBeenCalledTimes(1);
+ expect(dtCtor.ext.search).toEqual([]);
const fallbackInput = document.querySelector('#table-prs thead tr:nth-child(2) input');
fallbackInput.value = 'none';
@@ -442,6 +517,73 @@ describe('dashboard.js', () => {
await mod.loadDashboard();
});
+ test('organization selection scopes same-name repos, PR links and history, clearing empty charts', async () => {
+ document.body.insertAdjacentHTML('beforeend', `
+
`);
+ const repos = ['Second', 'First'].map(owner => ({
+ ...sampleRepos()[0], name: `${owner}/demo`, owner,
+ }));
+ repos.push({ ...sampleRepos()[0], name: 'Empty/archived', owner: 'Empty', archived: true });
+ const data = {
+ 'repos.json': repos,
+ 'prs.json': repos.slice(0, 2).map(repo => ({ repo: repo.name, number: 1 })),
+ 'metadata.json': { updated_at: '2026-11-07', registry_fresh: true },
+ 'star_history.json': repos.map(repo => ({ repo: repo.name, date: '2026-11-01', stars: 10 })),
+ 'coverage_history.json': repos.map(repo => ({ repo: repo.name, date: '2026-11-01', coverage: 90 })),
+ 'code_scanning_history.json': repos.map(repo => ({ repo: repo.name, date: '2026-11-01', open: 1 })),
+ 'commit_activity.json': repos.map(repo => ({ repo: repo.name, week: '2026-11-01', total: 3 })),
+ };
+ globalThis.fetch.mockImplementation(async url => ({
+ ok: url.split('/').at(-1) in data,
+ json: async () => data[url.split('/').at(-1)],
+ }));
+ await mod.loadDashboard();
+ const select = document.getElementById('organization-filter');
+ expect(Array.from(select.options).map(option => option.textContent))
+ .toEqual(['All organizations', 'Empty', 'First', 'Second']);
+ expect(document.querySelectorAll('#table-prs tbody tr')).toHaveLength(2);
+ expect(document.querySelector('#table-prs a').href).toBe('https://github.com/Second/demo/pull/1');
+ globalThis.Plotly.newPlot.mockClear();
+ select.value = 'First';
+ select.dispatchEvent(new Event('change'));
+ expect(document.querySelectorAll('#table-prs tbody tr')).toHaveLength(1);
+ expect(document.querySelector('#table-prs a').href).toBe('https://github.com/First/demo/pull/1');
+ for (const id of ['chart-star-history', 'chart-coverage-history', 'chart-code-scanning-history',
+ 'chart-commit-activity-repos']) {
+ const [, traces] = globalThis.Plotly.newPlot.mock.calls.find(([chart]) => chart === id);
+ expect(traces.map(trace => trace.name)).toEqual(['demo']);
+ expect(traces.map(trace => trace.meta)).toEqual(['First/demo']);
+ }
+ expect(document.getElementById('collection-status').hidden).toBe(true);
+ select.value = 'Empty';
+ document.getElementById('chart-stars').textContent = 'old chart';
+ select.dispatchEvent(new Event('change'));
+ expect(document.querySelectorAll('#table-prs tbody tr')).toHaveLength(0);
+ expect(document.getElementById('chart-stars').textContent).toBe('');
+ expect(globalThis.Plotly.purge).toHaveBeenCalledWith(document.getElementById('chart-stars'));
+ expect(globalThis.Plotly.purge).not.toHaveBeenCalledWith(document.getElementById('chart-azure-signing'));
+ });
+
+ test('organization controls tolerate legacy records and report partial collections', () => {
+ document.body.insertAdjacentHTML('beforeend', `
+
`);
+ mod.configureOrganizations([...sampleRepos(), { owner: 'Zeta' }, { owner: 'alpha' }, { owner: 'alpha' }],
+ jest.fn());
+ expect(Array.from(document.querySelectorAll('#organization-filter option')).map(option => option.textContent))
+ .toEqual(['All organizations', 'alpha', 'Zeta']);
+ const status = document.getElementById('collection-status');
+ mod.renderCollectionStatus({});
+ expect(status.hidden).toBe(true);
+ mod.renderCollectionStatus({ registry_fresh: false, organization_status: [
+ { name: 'First', status: 'stale' }, { name: 'Second', status: 'ok', codecov_status: 'stale' },
+ { name: 'Third', status: 'ok', codecov_status: 'ok' },
+ ] });
+ expect(status.hidden).toBe(false);
+ expect(status.textContent).toContain('organization list could not be refreshed');
+ expect(status.textContent).toContain('First, Second');
+ expect(status.textContent).not.toContain('Third');
+ });
+
test('DOMContentLoaded listener runs loadDashboard', async () => {
globalThis.fetch.mockResolvedValue({ ok: false, status: 500 });
document.dispatchEvent(new Event('DOMContentLoaded'));
diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py
index d038648401..d68cdf8361 100644
--- a/tests/unit/test_builder.py
+++ b/tests/unit/test_builder.py
@@ -14,8 +14,10 @@ def _write_json(path, payload):
path.write_text(json.dumps(payload), encoding='utf-8')
-def test_safe_name_uses_basename():
- assert builder._safe_name('x/y/z/repo') == 'repo'
+def test_safe_name_preserves_owner_and_rejects_arbitrary_paths():
+ assert builder._safe_name('owner/repo').replace('\\', '/') == 'owner/repo'
+ with pytest.raises(ValueError):
+ builder._safe_name('x/y/z/repo')
def test_load_rtd_repos_handles_missing_and_errors(tmp_path):
@@ -33,7 +35,7 @@ def test_load_rtd_repos_parses_repo_names(tmp_path):
{'repository': {'url': 'https://github.com/LizardByte/demo.git'}},
{'repository': {'url': 'https://github.com/LizardByte/demo2'}},
])
- assert builder._load_rtd_repos(str(tmp_path)) == {'demo', 'demo2'}
+ assert builder._load_rtd_repos(str(tmp_path)) == {'lizardbyte/demo', 'lizardbyte/demo2'}
def test_get_coverage_and_history(tmp_path):
diff --git a/tests/unit/test_catalog.py b/tests/unit/test_catalog.py
new file mode 100644
index 0000000000..d5a6d43fd9
--- /dev/null
+++ b/tests/unit/test_catalog.py
@@ -0,0 +1,267 @@
+"""Repository transfers must preserve published paths and history without conflating same-name projects."""
+
+import hashlib
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from src import catalog, builder, updater
+
+
+def write(path, data):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data), encoding='utf-8')
+
+
+def repository(identity, full_name, **values):
+ return SimpleNamespace(name=full_name.split('/')[1], owner=SimpleNamespace(login=full_name.split('/')[0]),
+ raw_data={'id': identity, 'name': full_name.split('/')[1], 'full_name': full_name,
+ 'private': False, 'archived': False, **values})
+
+
+def client(groups):
+ def organization(name):
+ def repos(**kwargs):
+ assert kwargs == {'type': 'public'}
+ if isinstance(groups[name], Exception):
+ raise groups[name]
+ return iter(groups[name])
+ return SimpleNamespace(get_repos=repos)
+ return SimpleNamespace(get_organization=organization)
+
+
+def registry(monkeypatch, names, status=200):
+ def get(**kwargs):
+ assert kwargs == {'url': catalog.ORGANIZATIONS_URL, 'timeout': 30}
+ return SimpleNamespace(status_code=status, json=lambda: names)
+ monkeypatch.setattr(catalog.helpers.s, 'get', get)
+
+
+def test_transfer_keeps_original_urls_and_cache_bytes(monkeypatch, tmp_path):
+ old = repository(17, 'LizardByte/Sunshine').raw_data
+ write(tmp_path / 'github/repos.json', [old])
+ histories = ['commitActivity', 'commitActivityHashes', 'starHistory', 'codeScanningHistory', 'prMetrics']
+ for kind in histories:
+ write(tmp_path / 'github' / kind / 'Sunshine.json', [{'historical': kind}])
+ banner = tmp_path / 'github/openGraphImages/Sunshine.png'
+ banner.parent.mkdir(parents=True)
+ banner.write_bytes(b'historical-banner')
+ write(tmp_path / 'shields/codecov/Sunshine.json', {'message': '97%'})
+ paths = [path for path in tmp_path.rglob('*') if path.is_file() and path.name != 'repos.json']
+ checksums = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in paths}
+ registry(monkeypatch, ['LizardByte', 'SunshineStream'])
+ moved = repository(17, 'SunshineStream/Sunshine')
+ _, state = catalog.discover(client({'LizardByte': [], 'SunshineStream': [moved]}), str(tmp_path))
+ assert state['repositories']['17']['key'] == 'Sunshine'
+ assert catalog.cache_key(moved) == 'Sunshine'
+ data = json.loads((tmp_path / 'github/repos.json').read_text(encoding='utf-8'))[0]
+ assert data['dashboard_paths']['banner'] == 'github/openGraphImages/Sunshine.png'
+ assert data['dashboard_paths']['coverage_badge'] == 'shields/codecov/Sunshine.json'
+ assert data['full_name'] == 'SunshineStream/Sunshine'
+ assert data['dashboard_aliases'] == ['LizardByte/Sunshine', 'SunshineStream/Sunshine']
+ assert checksums == {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in paths}
+ assert catalog.documentation_key(str(tmp_path), 'git@github.com:LizardByte/Sunshine.git') == 'Sunshine'
+ # A later collection must retain the same identity without depending on the original catalog location.
+ catalog.discover(client({'LizardByte': [], 'SunshineStream': [moved]}), str(tmp_path))
+ assert catalog.cache_key(moved) == 'Sunshine'
+
+
+def test_distinct_orgs_with_same_names_keep_independent_storage_and_private_repos_stay_private(monkeypatch, tmp_path):
+ registry(monkeypatch, ['First', 'Second'])
+ first, second = repository(1, 'First/demo'), repository(2, 'Second/demo', archived=True)
+ private = repository(3, 'Second/private', private=True, description='never publish this')
+ objects, state = catalog.discover(client({'First': [first], 'Second': [second, private]}), str(tmp_path))
+ assert len(objects) == 2
+ assert catalog.cache_key(first) == 'First/demo'
+ assert catalog.cache_key(second) == 'Second/demo'
+ assert catalog.cache_key({'name': 'old'}) == 'old'
+ serialized = (tmp_path / 'github/repos.json').read_text(encoding='utf-8')
+ assert 'never publish' not in serialized
+ assert '3' not in state['repositories']
+ assert json.loads(serialized)[1]['archived']
+
+
+def test_failed_org_preserves_snapshot_but_fresh_owner_wins_for_transferred_id(monkeypatch, tmp_path):
+ write(tmp_path / 'github/repos.json', [
+ repository(1, 'First/moved').raw_data, repository(2, 'First/stays').raw_data])
+ write(tmp_path / 'github/catalog.json', {'version': 1, 'repositories': {}, 'organizations': [
+ {'name': 'First', 'status': 'ok', 'collected_at': 'original-time'}]})
+ registry(monkeypatch, ['First', 'Second', 'Empty'])
+ fresh = repository(1, 'Second/moved')
+ _, state = catalog.discover(client({'First': RuntimeError('offline'), 'Second': [fresh],
+ 'Empty': RuntimeError('no access')}), str(tmp_path))
+ records = json.loads((tmp_path / 'github/repos.json').read_text(encoding='utf-8'))
+ assert {r['id']: r['full_name'] for r in records} == {1: 'Second/moved', 2: 'First/stays'}
+ assert state['organizations'][0] == {'name': 'First', 'status': 'stale', 'collected_at': 'original-time'}
+ assert state['organizations'][2]['status'] == 'unavailable'
+
+
+@pytest.mark.parametrize('names', [[], {}, ['bad/name'], ['First', 'first'], [1]])
+def test_invalid_registry_cannot_replace_valid_state(monkeypatch, names):
+ registry(monkeypatch, names)
+ with pytest.raises(ValueError):
+ catalog.organizations({'organizations': []})
+ assert catalog.organizations({'organizations': [{'name': 'Previous'}]}) == (['Previous'], False)
+
+
+def test_registry_outage_uses_last_registry_only_if_available(monkeypatch):
+ registry(monkeypatch, [], status=503)
+ assert catalog.organizations({'organizations': [{'name': 'Cached'}]}) == (['Cached'], False)
+ with pytest.raises(ValueError, match='request failed'):
+ catalog.organizations({'organizations': []})
+
+
+@pytest.mark.parametrize('key', ['', '.', '..', '../repo', '/repo', 'a/b/c', r'..\repo'])
+def test_invalid_keys_cannot_escape_cache_root(key):
+ with pytest.raises(ValueError, match='cache key'):
+ catalog.safe_key(key)
+
+
+def test_index_validation_rejects_ambiguous_legacy_paths_and_unknown_versions(tmp_path):
+ write(tmp_path / 'github/repos.json', [
+ repository(1, 'First/demo').raw_data, repository(2, 'Second/demo').raw_data])
+ with pytest.raises(ValueError, match='conflicting identities'):
+ catalog.load_state(str(tmp_path))
+ write(tmp_path / 'github/catalog.json', {'version': 2})
+ with pytest.raises(ValueError, match='Unsupported'):
+ catalog.load_state(str(tmp_path))
+ conflicting = repository(2, 'First/demo').raw_data
+ with pytest.raises(ValueError, match='another repository ID'):
+ catalog._register({'repositories': {'1': {'key': 'First/demo'}}}, conflicting)
+
+
+def test_legacy_private_entries_are_not_registered_and_non_github_docs_are_not_guessed(tmp_path):
+ write(tmp_path / 'github/repos.json', [repository(1, 'First/private', private=True).raw_data])
+ state, _ = catalog.load_state(str(tmp_path))
+ assert state['repositories'] == {}
+ assert catalog.github_name('https://example.com/First/demo.git') is None
+ assert catalog.documentation_key(str(tmp_path), 'https://github.com/Other/demo.git') == 'Other/demo'
+ with pytest.raises(ValueError, match='no GitHub'):
+ catalog.documentation_key(str(tmp_path), 'https://example.com/First/demo.git')
+
+
+def test_builder_reads_distinct_keys_and_publishes_canonical_links_and_legacy_report_alias(monkeypatch, tmp_path):
+ base, template = tmp_path / 'cache', tmp_path / 'site'
+ records = [dict(repository(1, 'New/demo').raw_data, dashboard_key='demo', dashboard_aliases=['Old/demo']),
+ dict(repository(2, 'Other/demo').raw_data, dashboard_key='Other/demo')]
+ write(base / 'github/repos.json', records)
+ write(base / 'github/languages/demo.json', {'Python': 7})
+ write(base / 'github/languages/Other/demo.json', {'Rust': 9})
+ write(base / 'github/pulls/demo.json', [{'number': 1, 'title': 'first'}])
+ write(base / 'github/pulls/Other/demo.json', [{'number': 1, 'title': 'second'}])
+ write(base / 'github/starHistory/demo.json', [{'date': '2025-01-01', 'stars': 11}])
+ write(base / 'readthedocs/projects.json', [{'repository': {'url': 'https://github.com/Old/demo.git'}}])
+ monkeypatch.setattr(builder, 'BASE_DIR', str(base))
+ monkeypatch.setattr(builder, 'TEMPLATE_DIR', str(template))
+ builder.build()
+ data = json.loads((template / 'assets/data/repos.json').read_text(encoding='utf-8'))
+ assert [(repo['name'], repo['languages'], repo['has_readthedocs']) for repo in data] == [
+ ('New/demo', {'Python': 7}, True), ('Other/demo', {'Rust': 9}, False)]
+ pulls = json.loads((template / 'assets/data/prs.json').read_text(encoding='utf-8'))
+ assert [(pull['repo'], pull['number']) for pull in pulls] == [('New/demo', 1), ('Other/demo', 1)]
+ assert json.loads((template / 'assets/data/star_history.json').read_text(encoding='utf-8'))[0]['repo'] == 'New/demo'
+ assert 'permalink: /pr-metrics/demo/' in (template / 'pr-metrics/demo.md').read_text(encoding='utf-8')
+ assert 'repo%3ANew%2Fdemo' in (template / 'pr-metrics/New/demo.md').read_text(encoding='utf-8')
+ index = (template / 'pr-metrics/index.md').read_text(encoding='utf-8')
+ assert 'org%3ANew+org%3AOther' in index
+
+
+def test_transferred_star_and_commit_caches_still_resolve_under_original_key(monkeypatch, tmp_path):
+ repo = repository(17, 'New/renamed')
+ repo.dashboard_key = 'old-name'
+ repo.stargazers_count = 9
+ write(tmp_path / 'github/starHistory/old-name.json', [{'date': '2024-01-01', 'stars': 5}])
+ write(tmp_path / 'github/commitActivityHashes/old-name.json', {'sha': 'unchanged'})
+ monkeypatch.setattr(updater, 'BASE_DIR', str(tmp_path))
+ assert updater._collect_star_history(repo)[0] == {'date': '2024-01-01', 'stars': 5}
+ assert updater._cached_commit_activity_sha(repo) == 'unchanged'
+
+
+def test_coverage_transfer_keeps_old_points_and_updates_overlaps(tmp_path):
+ path = tmp_path / 'codecov/old-name_coverage_trend'
+ write(path.with_suffix('.json'), [
+ {'timestamp': '2024-01-01', 'avg': 70}, {'timestamp': '2025-01-01', 'avg': 80}])
+ assert updater._merge_coverage_history(str(path), [
+ {'timestamp': '2025-01-01', 'avg': 85}, {'timestamp': '2026-01-01', 'avg': 90}]) == [
+ {'timestamp': '2024-01-01', 'avg': 70}, {'timestamp': '2025-01-01', 'avg': 85},
+ {'timestamp': '2026-01-01', 'avg': 90}]
+
+
+def test_codecov_each_org_uses_its_owner_and_stable_key_and_retains_failed_data(monkeypatch, tmp_path):
+ records = [dict(repository(1, 'New/demo').raw_data, dashboard_key='legacy-demo'),
+ dict(repository(2, 'Other/demo').raw_data, dashboard_key='Other/demo'),
+ dict(repository(3, 'New/archived', archived=True).raw_data, dashboard_key='old-archived')]
+ write(tmp_path / 'github/repos.json', records)
+ write(tmp_path / 'github/catalog.json', {'organizations': [{'name': 'New'}, {'name': 'Other'}, {'name': 'Failed'}]})
+ write(tmp_path / 'codecov/Other/demo.json', {'totals': {'coverage': 71}})
+ write(tmp_path / 'codecov/legacy-demo_coverage_trend.json', [{'timestamp': '2024-01-01', 'avg': 70}])
+ calls = []
+
+ def get(url, headers):
+ calls.append(url)
+ status = 200
+ if '/Failed/' in url:
+ data, status = {}, 403
+ elif url.endswith('/repos?page_size=500'):
+ data = {'next': None, 'results': [{'name': 'demo'}, {'name': 'private'}, {'name': 'archived'}]}
+ elif '/Other/' in url:
+ data, status = {}, 404
+ else:
+ data = {'totals': {'coverage': 92}}
+ return SimpleNamespace(status_code=status, json=lambda: data)
+
+ monkeypatch.setattr(updater, 'BASE_DIR', str(tmp_path))
+ monkeypatch.setenv('CODECOV_TOKEN', 'test-only')
+ monkeypatch.setattr(updater.helpers.s, 'get', get)
+ monkeypatch.setattr(updater, 'fetch_coverage_trend_for_repo',
+ lambda *args: [{'timestamp': '2026-01-01', 'avg': 92}])
+ updater.update_codecov()
+ assert json.loads((tmp_path / 'shields/codecov/legacy-demo.json').read_text())['message'] == '92%'
+ assert json.loads((tmp_path / 'codecov/Other/demo.json').read_text())['totals']['coverage'] == 71
+ assert json.loads((tmp_path / 'codecov/legacy-demo_coverage_trend.json').read_text()) == [
+ {'timestamp': '2024-01-01', 'avg': 70}, {'timestamp': '2026-01-01', 'avg': 92}]
+ state = json.loads((tmp_path / 'github/catalog.json').read_text())
+ assert [org['codecov_status'] for org in state['organizations']] == ['ok', 'stale', 'stale']
+ assert not any(url.endswith('/private') or url.endswith('/archived') for url in calls)
+
+
+def test_codecov_repository_pagination_stays_inside_the_source_org(monkeypatch):
+ base = 'https://codecov.io/api/v2/github/First'
+ responses = iter([{'next': base + '/repos?page=2', 'results': [{'name': 'one'}]},
+ {'next': None, 'results': [{'name': 'two'}]}])
+ monkeypatch.setattr(updater.helpers.s, 'get',
+ lambda **kwargs: SimpleNamespace(status_code=200, json=lambda: next(responses)))
+ assert list(updater._codecov_repositories(base, {})) == [{'name': 'one'}, {'name': 'two'}]
+
+
+def test_codecov_inactive_project_is_not_an_outage_and_collects_after_its_first_upload(monkeypatch, tmp_path):
+ write(tmp_path / 'github/repos.json', [dict(repository(1, 'New/demo').raw_data, dashboard_key='old-name')])
+ write(tmp_path / 'github/catalog.json', {'organizations': [{'name': 'New', 'status': 'ok'}]})
+ cache = tmp_path / 'codecov/old-name.json'
+ write(cache, {'totals': {'coverage': 71}})
+ previous = cache.read_bytes()
+ calls, active = [], False
+
+ def get(url, headers):
+ calls.append(url)
+ if url.endswith('/repos?page_size=500'):
+ data, status = {'next': None, 'results': [{'name': 'demo', 'active': active}]}, 200
+ else:
+ data, status = ({'totals': {'coverage': 92}}, 200) if active else ({}, 404)
+ return SimpleNamespace(status_code=status, json=lambda: data)
+
+ monkeypatch.setattr(updater, 'BASE_DIR', str(tmp_path))
+ monkeypatch.setenv('CODECOV_TOKEN', 'test-only')
+ monkeypatch.setattr(updater.helpers.s, 'get', get)
+ monkeypatch.setattr(updater, 'fetch_coverage_trend_for_repo', lambda *args: [])
+ updater.update_codecov()
+ state = json.loads((tmp_path / 'github/catalog.json').read_text())
+ assert state['organizations'][0]['codecov_status'] == 'ok'
+ assert calls == ['https://codecov.io/api/v2/github/New/repos?page_size=500']
+ assert cache.read_bytes() == previous
+
+ active = True
+ updater.update_codecov()
+ assert json.loads(cache.read_text())['totals']['coverage'] == 92
+ assert calls[-1] == 'https://codecov.io/api/v2/github/New/repos/demo'
diff --git a/tests/unit/test_pr_metrics.py b/tests/unit/test_pr_metrics.py
index f87af9693c..55c0bcdd88 100644
--- a/tests/unit/test_pr_metrics.py
+++ b/tests/unit/test_pr_metrics.py
@@ -104,7 +104,7 @@ def test_datetime_and_active_repo_helpers():
def test_cache_helpers(monkeypatch, tmp_path):
now = datetime(2026, 3, 20, tzinfo=timezone.utc)
- assert pr_metrics.cache_path(str(tmp_path), 'x/demo').endswith(('prMetrics\\demo', 'prMetrics/demo'))
+ assert pr_metrics.cache_path(str(tmp_path), 'x/demo').endswith(('prMetrics\\x\\demo', 'prMetrics/x/demo'))
assert pr_metrics.load_cache(str(tmp_path), 'demo') is None
assert not pr_metrics.cache_is_fresh(None, now)
diff --git a/tests/unit/test_updater.py b/tests/unit/test_updater.py
index 9b5efef10b..4990e4475b 100644
--- a/tests/unit/test_updater.py
+++ b/tests/unit/test_updater.py
@@ -241,7 +241,7 @@ def test_update_codecov_error_paths(monkeypatch, tmp_path):
'get',
lambda url, headers: FakeResponse({'next': 'more', 'results': []}, 200)
)
- with pytest.raises(AssertionError):
+ with pytest.raises(ValueError, match='Invalid Codecov pagination URL'):
updater.update_codecov()
assert warnings
@@ -619,21 +619,24 @@ def test_process_github_repo_error_and_avatar_skip(monkeypatch, tmp_path):
assert any('OpenGraph image URL' in warning for warning in warnings)
-def test_update_github(monkeypatch):
+def test_update_github(monkeypatch, tmp_path):
monkeypatch.setenv('GITHUB_TOKEN', 'tok')
monkeypatch.setenv('GITHUB_REPOSITORY_OWNER', 'owner')
repo_active = FakeRepo('active', archived=False)
repo_pending = FakeRepo('pending', archived=False)
repo_archived = FakeRepo('archived', archived=True)
+ for identity, repo in enumerate([repo_active, repo_pending, repo_archived], 1):
+ repo.raw_data.update({'id': identity, 'full_name': f'owner/{repo.name}'})
+ monkeypatch.setattr(updater.helpers.s, 'get', lambda **kwargs: FakeResponse(['owner']))
- owner = SimpleNamespace(get_repos=lambda: [repo_active, repo_pending, repo_archived])
+ owner = SimpleNamespace(get_repos=lambda **kwargs: [repo_active, repo_pending, repo_archived])
class FakeGithub:
def __init__(self, auth, timeout):
self.per_page = None
- def get_user(self, name):
+ def get_organization(self, name):
return owner
monkeypatch.setattr(updater, 'Github', FakeGithub)
@@ -655,7 +658,7 @@ def get_user(self, name):
)
processed = []
monkeypatch.setattr(updater, '_process_github_repo', lambda repo, headers, graphql_url: processed.append(repo.name))
- monkeypatch.setattr(updater, 'BASE_DIR', 'base')
+ monkeypatch.setattr(updater, 'BASE_DIR', str(tmp_path))
updater.update_github()
@@ -788,6 +791,27 @@ def __init__(self, state):
assert result == alerts
+@pytest.mark.parametrize('status', [403, 404])
+def test_process_github_repo_preserves_code_scanning_history_when_unavailable(monkeypatch, tmp_path, status):
+ monkeypatch.setattr(updater, 'BASE_DIR', str(tmp_path))
+ repo = FakeRepo(name='demo')
+ repo.dashboard_key = 'old-name'
+ paths = [tmp_path / 'github/codeScanning/old-name.json', tmp_path / 'github/codeScanningHistory/old-name.json']
+ for path, data in zip(paths, [{'open': 3}, [{'date': '2025-01-01', 'open': 3}]]):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data), encoding='utf-8')
+ previous = [path.read_bytes() for path in paths]
+
+ def unavailable():
+ raise GithubException(status=status, data={'message': 'unavailable'})
+
+ monkeypatch.setattr(repo, 'get_codescan_alerts', unavailable)
+ monkeypatch.setattr(updater, '_collect_star_history', lambda repo: [])
+ monkeypatch.setattr(updater, '_fetch_open_graph_image_url', lambda *args: None)
+ updater._process_github_repo(repo, {}, 'unused')
+ assert [path.read_bytes() for path in paths] == previous
+
+
def test_fetch_code_scanning_alerts_404_skip(monkeypatch):
repo = FakeRepo(name='demo')
@@ -799,7 +823,7 @@ def raise_404():
monkeypatch.setattr(repo, 'get_codescan_alerts', lambda **kwargs: raise_404())
- assert updater._fetch_code_scanning_alerts(repo) == []
+ assert updater._fetch_code_scanning_alerts(repo) is None
assert warnings