Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/collect.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ jobs:
git config user.name "pulse-bot"
git config user.email "[email protected]"
git commit -m "data: collect $(date -u +%Y-%m-%d)"
# main may have moved while collecting (a merge mid-run); data/
# is only written by this job, so rebasing onto it is safe.
git pull --rebase origin main
git push
# A push made with GITHUB_TOKEN does not trigger other workflows,
# so redeploy the dashboard explicitly.
Expand Down
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ site/ see site/AGENTS.md
| repos/stars | `date,stars` | cumulative; backfilled once from the stargazers API (timestamp per star), then today's count from the health snapshot |
| docs/traffic | `date,host,pageviews,visits` | daily, HTML 200s to browsers at the Cloudflare edge; refetched for 30 days each run |
| docs/pages | `date,path,pageviews` | daily snapshot of the top 200 paths over the last 7 days |
| docs/referrers | `date,referrer,pageviews` | daily, page views by referrer host; the site's own hosts dropped, `(direct)` for none; refetched for 30 days |
| docs/referrers | `date,referrer,pageviews` | daily, page views by referrer host; the site's own hosts dropped, `(direct)` for none; refetched for 30 days; skipped with a note when the zone plan does not expose referrers |
| docs/countries | `date,country,pageviews` | daily, page views by visitor country (ISO code); refetched for 30 days |
| repos/contributors | `date,author,commits` | per author per day on the default branch; bots excluded; author is a login or `email:<address>`; backfilled once, then the latest month refetched |

Expand Down
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,12 +144,15 @@ The same HTML page views, per referrer host per day. Referrers that are
the site's own hosts (clicks from one page to another) are dropped, so a
row counts arrivals from elsewhere. `(direct)` is no referrer at all:
typed links, bookmarks, and apps like Slack that hide the referrer.
Cloudflare only exposes the referrer field on some zone plans. The
nebari.dev zone refuses it today, so the collector skips referrers with a
note and this file is not written.

**Countries** (`docs/countries/<site>.csv`, `date,country,pageviews`)
The same page views per visitor country, as an ISO code (`XX` is
unknown, `T1` is Tor). Referrers and countries come from a second query
and are refetched for 30 days like traffic; if that query fails, traffic
and top pages are still kept.
unknown, `T1` is Tor). Referrers and countries are one query each,
separate from traffic, and are refetched for 30 days like traffic. If one
fails, the other and traffic and top pages are still kept.

### Derived numbers

Expand Down
2 changes: 2 additions & 0 deletions nebari_pulse/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ def _collect(cfg: config.Config, args: argparse.Namespace, today) -> int:
for res in results:
if res.source == "cloudflare_docs" and res.notes.get("skipped"):
print(f"docs traffic skipped: {res.notes['skipped']}")
if res.source == "cloudflare_docs" and res.notes.get("referrers_unavailable"):
print("docs referrers skipped: the zone's Cloudflare plan does not expose them")
if res.source == "github_repos" and res.notes.get("traffic_denied"):
denied = res.notes["traffic_denied"]
print(
Expand Down
107 changes: 54 additions & 53 deletions nebari_pulse/sources/cloudflare_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
visitor country (ISO code) per day.
Both refetched for the last 30 days like traffic.

Referrers and countries come from a second query, so a failure there is
reported without losing traffic and top pages.
Referrers and countries are one query each, separate from traffic, so a
failure in one is reported without losing the others. Cloudflare only
exposes the referrer field on some zone plans; when the zone refuses it,
referrers are skipped with a note instead of an error on every run.

Needs an API token with Zone:Read (to resolve the zone id by name) and
Analytics:Read on the zone: CLOUDFLARE_ANALYTICS_TOKEN, or the deploy
Expand Down Expand Up @@ -91,11 +93,11 @@
"""


ORIGINS_QUERY = """
ORIGIN_QUERY = """
query ($zone: String!, $hosts: [String!], $start: Date!, $end: Date!) {
viewer {
zones(filter: { zoneTag: $zone }) {
referrers: httpRequestsAdaptiveGroups(
%s: httpRequestsAdaptiveGroups(
limit: 10000
orderBy: [count_DESC]
filter: {
Expand All @@ -107,27 +109,18 @@
}
) {
count
dimensions { date clientRefererHost }
}
countries: httpRequestsAdaptiveGroups(
limit: 10000
orderBy: [count_DESC]
filter: {
date_geq: $start, date_leq: $end
requestSource: "eyeball"
clientRequestHTTPHost_in: $hosts
edgeResponseContentTypeName: "html"
edgeResponseStatus: 200
}
) {
count
dimensions { date clientCountryName }
dimensions { date %s }
}
}
}
}
"""

REFERRERS_QUERY = ORIGIN_QUERY % ("referrers", "clientRefererHost")
COUNTRIES_QUERY = ORIGIN_QUERY % ("countries", "clientCountryName")
# Text of the GraphQL error when the zone plan does not expose a field.
FIELD_DENIED = "does not have access to the field"


def token() -> str | None:
"""CLOUDFLARE_ANALYTICS_TOKEN, or the deploy token (CLOUDFLARE_API_TOKEN)
Expand Down Expand Up @@ -205,42 +198,50 @@ def collect(cfg: DocsConfig, data_dir: Path, today: date) -> CollectResult:
def collect_origins(
zone: str, site: DocsSite, start: date, end: date, data_dir: Path, result: CollectResult
) -> int:
"""Referrer and country rows for one site; a failure is a partial error."""
try:
payload = post_json(
GRAPHQL,
{
"query": ORIGINS_QUERY,
"variables": {
"zone": zone,
"hosts": list(site.hosts),
"start": start.isoformat(),
"end": end.isoformat(),
},
},
headers=headers(),
"""Referrer and country rows for one site. Each is its own query, so one
failing keeps the other; a failure is a partial error, except a field the
zone plan does not expose, which is a note."""
variables = {
"zone": zone,
"hosts": list(site.hosts),
"start": start.isoformat(),
"end": end.isoformat(),
}
zone_data: dict[str, list] = {}
for key, query in (("referrers", REFERRERS_QUERY), ("countries", COUNTRIES_QUERY)):
try:
payload = post_json(
GRAPHQL, {"query": query, "variables": variables}, headers=headers()
)
except HttpError as e:
result.errors.append(f"{site.id} {key}: {e}")
continue
if payload.get("errors"):
message = payload["errors"][0].get("message") or ""
if FIELD_DENIED in message:
result.notes[f"{key}_unavailable"] = message
else:
result.errors.append(f"{site.id} {key}: {message}")
continue
zones = (payload.get("data") or {}).get("viewer", {}).get("zones") or []
if zones:
zone_data[key] = zones[0].get(key) or []
referrers, countries = parse_origins({"data": {"viewer": {"zones": [zone_data]}}}, site.hosts)
changed = 0
if "referrers" in zone_data:
changed += store.upsert(
store.path_for(data_dir, "docs", "referrers", site.id),
REFERRER_COLUMNS,
["date", "referrer"],
referrers,
)
except HttpError as e:
result.errors.append(f"{site.id} referrers/countries: {e}")
return 0
if payload.get("errors"):
result.errors.append(
f"{site.id} referrers/countries: {payload['errors'][0].get('message')}"
if "countries" in zone_data:
changed += store.upsert(
store.path_for(data_dir, "docs", "countries", site.id),
COUNTRY_COLUMNS,
["date", "country"],
countries,
)
return 0
referrers, countries = parse_origins(payload, site.hosts)
changed = store.upsert(
store.path_for(data_dir, "docs", "referrers", site.id),
REFERRER_COLUMNS,
["date", "referrer"],
referrers,
)
changed += store.upsert(
store.path_for(data_dir, "docs", "countries", site.id),
COUNTRY_COLUMNS,
["date", "country"],
countries,
)
log.info("docs %s: %d referrer-days, %d country-days", site.id, len(referrers), len(countries))
return changed

Expand Down
2 changes: 1 addition & 1 deletion site/src/pages/Docs/Docs.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ describe('Docs', () => {
const referrers = await screen.findByRole('table', {
name: /Where readers come from/,
});
expect(referrers).toHaveTextContent('No referrer data yet.');
expect(referrers).toHaveTextContent('No referrer data.');
});

it('filters the table by site from the URL', async () => {
Expand Down
2 changes: 1 addition & 1 deletion site/src/pages/Docs/Docs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ function Docs() {
column="Referrer"
rank={view.referrers}
label={(name) => (name === '(direct)' ? 'Direct or unknown' : name)}
empty="No referrer data yet. It fills in from the next daily run."
empty="No referrer data. Cloudflare does not report referrers on this zone's plan."
info={
<ul>
<li>
Expand Down
153 changes: 127 additions & 26 deletions tests/test_cloudflare_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,44 +135,145 @@ def test_parse_origins_drops_internal_navigation():
assert cloudflare_docs.parse_origins({"data": {"viewer": {"zones": []}}}, ()) == ([], [])


def test_origins_failure_keeps_traffic(tmp_path, monkeypatch):
from nebari_pulse.config import DocsConfig, DocsSite
from nebari_pulse.http import HttpError
TRAFFIC_PAYLOAD = {
"data": {
"viewer": {
"zones": [
{
"traffic": [
{
"count": 3,
"sum": {"visits": 1},
"dimensions": {
"date": "2026-10-22",
"clientRequestHTTPHost": "nebari.dev",
},
}
],
"pages": [],
}
]
}
}
}

monkeypatch.setenv("CLOUDFLARE_ANALYTICS_TOKEN", "t")
monkeypatch.setattr(cloudflare_docs, "zone_id", lambda name: "zone")
COUNTRIES_PAYLOAD = {
"data": {
"viewer": {
"zones": [
{
"countries": [
{
"count": 4,
"dimensions": {"date": "2026-10-22", "clientCountryName": "DE"},
}
]
}
]
}
}
}

def fake_post(url, body, headers):
if body["query"] is cloudflare_docs.ORIGINS_QUERY:
raise HttpError(url, 400, "unknown field clientRefererHost")
return {
"data": {
"viewer": {
"zones": [
REFERRERS_PAYLOAD = {
"data": {
"viewer": {
"zones": [
{
"referrers": [
{
"traffic": [
{
"count": 3,
"sum": {"visits": 1},
"dimensions": {
"date": "2026-10-22",
"clientRequestHTTPHost": "nebari.dev",
},
}
],
"pages": [],
"count": 2,
"dimensions": {
"date": "2026-10-22",
"clientRefererHost": "github.com",
},
}
]
}
}
]
}
}
}


def run_docs(tmp_path, monkeypatch, responses):
"""Collect one site with post_json answered per query from `responses`."""
from nebari_pulse.config import DocsConfig, DocsSite

monkeypatch.setenv("CLOUDFLARE_ANALYTICS_TOKEN", "t")
monkeypatch.setattr(cloudflare_docs, "zone_id", lambda name: "zone")

def fake_post(url, body, headers):
answer = responses[body["query"]]
if isinstance(answer, Exception):
raise answer
return answer

monkeypatch.setattr(cloudflare_docs, "post_json", fake_post)
cfg = DocsConfig(
cloudflare_zone="nebari.dev", sites=(DocsSite("docs", "Docs", ("nebari.dev",)),)
)
res = cloudflare_docs.collect(cfg, tmp_path, TODAY)
return cloudflare_docs.collect(cfg, tmp_path, TODAY)


def test_origins_failure_keeps_traffic(tmp_path, monkeypatch):
from nebari_pulse.http import HttpError

failure = HttpError("url", 500, "boom")
res = run_docs(
tmp_path,
monkeypatch,
{
cloudflare_docs.QUERY: TRAFFIC_PAYLOAD,
cloudflare_docs.REFERRERS_QUERY: failure,
cloudflare_docs.COUNTRIES_QUERY: failure,
},
)
assert res.artifacts == 1
assert len(res.errors) == 1 and "referrers/countries" in res.errors[0]
assert len(res.errors) == 2
assert (tmp_path / "docs" / "traffic" / "docs.csv").exists()
assert not (tmp_path / "docs" / "referrers" / "docs.csv").exists()
assert not (tmp_path / "docs" / "countries" / "docs.csv").exists()


def test_referrer_field_denied_is_a_note_and_keeps_countries(tmp_path, monkeypatch):
denied = {
"data": None,
"errors": [
{
"message": "zone 'abc' does not have access to the field "
"'clientrefererhost' from the path"
}
],
}
res = run_docs(
tmp_path,
monkeypatch,
{
cloudflare_docs.QUERY: TRAFFIC_PAYLOAD,
cloudflare_docs.REFERRERS_QUERY: denied,
cloudflare_docs.COUNTRIES_QUERY: COUNTRIES_PAYLOAD,
},
)
assert res.errors == []
assert "referrers_unavailable" in res.notes
assert not (tmp_path / "docs" / "referrers" / "docs.csv").exists()
rows = (tmp_path / "docs" / "countries" / "docs.csv").read_text().splitlines()
assert rows == ["date,country,pageviews", "2026-10-22,DE,4"]


def test_country_failure_keeps_referrers(tmp_path, monkeypatch):
from nebari_pulse.http import HttpError

res = run_docs(
tmp_path,
monkeypatch,
{
cloudflare_docs.QUERY: TRAFFIC_PAYLOAD,
cloudflare_docs.REFERRERS_QUERY: REFERRERS_PAYLOAD,
cloudflare_docs.COUNTRIES_QUERY: HttpError("url", 500, "boom"),
},
)
assert len(res.errors) == 1 and "countries" in res.errors[0]
rows = (tmp_path / "docs" / "referrers" / "docs.csv").read_text().splitlines()
assert rows == ["date,referrer,pageviews", "2026-10-22,github.com,2"]
assert not (tmp_path / "docs" / "countries" / "docs.csv").exists()
Loading