Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion site/src/lib/coverage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,25 @@ import { coverageLines, partialNote } from './coverage';
import type { Coverage } from './summary';

const cov: Coverage = {
starts: [{ date: '2026-10-11', sources: ['conda', 'github_releases'] }],
starts: [{ date: '2026-10-11', keys: ['conda', 'github_releases'] }],
partialFrom: '2026-10-24',
lagging: ['quay', 'pypi'],
keys: ['quay', 'pypi', 'conda', 'github_releases'],
};

describe('coverage captions', () => {
it('says "Data starts" when every series starts at once', () => {
const all: Coverage<string> = {
starts: [{ date: 'd', keys: ['a', 'b'] }],
partialFrom: null,
lagging: [],
keys: ['a', 'b'],
};
expect(coverageLines(all, (k) => k.toUpperCase())).toEqual([
{ date: 'd', label: 'Data starts' },
]);
});

it('labels start lines and the incomplete tail', () => {
expect(coverageLines(cov)).toEqual([
{ date: '2026-10-11', label: 'conda-forge and GitHub Releases start' },
Expand Down
28 changes: 21 additions & 7 deletions site/src/lib/coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,33 @@ export function sourceList(sources: readonly Source[]): string {
return list.format(sources.map((s) => SOURCE_LABEL[s]));
}

/** "PyPI starts" etc., one line per date a source begins. */
export function coverageLines(
cov: Coverage,
const sourceLabel = (s: Source) => SOURCE_LABEL[s];

/**
* "PyPI starts" etc., one line per start; "Data starts" when every series
* begins there. `label` names a key (defaults to source names).
*/
export function coverageLines<K extends string>(
cov: Coverage<K>,
label: (key: K) => string = sourceLabel as (key: K) => string,
): Array<{ date: string; label: string }> {
return cov.starts.map(({ date, sources }) => ({
return cov.starts.map(({ date, keys }) => ({
date,
label: `${sourceList(sources)} ${sources.length > 1 ? 'start' : 'starts'}`,
label:
keys.length === cov.keys.length && keys.length > 1
? 'Data starts'
: `${list.format(keys.map(label))} ${keys.length > 1 ? 'start' : 'starts'}`,
}));
}

/** Tooltip note for the incomplete tail. */
export function partialNote(cov: Coverage, weekly: boolean): string {
export function partialNote<K extends string>(
cov: Coverage<K>,
weekly: boolean,
label: (key: K) => string = sourceLabel as (key: K) => string,
): string {
const unit = weekly ? 'week' : 'day';
if (cov.lagging.length === 0) return `This ${unit} is not over yet.`;
return `${sourceList(cov.lagging)} ${cov.lagging.length > 1 ? 'have' : 'has'} not reported this ${unit} yet, so it reads low.`;
const names = list.format(cov.lagging.map(label));
return `${names} ${cov.lagging.length > 1 ? 'have' : 'has'} not reported this ${unit} yet, so it reads low.`;
}
29 changes: 26 additions & 3 deletions site/src/lib/summary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
fillDaily,
platformOf,
type Source,
seriesCoverage,
shareRows,
sumSeries,
versionShareInputs,
Expand Down Expand Up @@ -318,8 +319,8 @@ describe('coverage', () => {
'2026-10-24',
);
expect(cov.starts).toEqual([
{ date: '2026-10-01', sources: ['pypi'] },
{ date: '2026-10-10', sources: ['conda', 'github_releases'] },
{ date: '2026-10-01', keys: ['pypi'] },
{ date: '2026-10-10', keys: ['conda', 'github_releases'] },
]);
expect(cov.lagging).toEqual(['quay', 'pypi']);
expect(cov.partialFrom).toBe('2026-10-24');
Expand All @@ -331,7 +332,29 @@ describe('coverage', () => {
'2026-09-25',
'2026-10-24',
);
expect(cov).toEqual({ starts: [], partialFrom: null, lagging: [] });
expect(cov).toEqual({
starts: [],
partialFrom: null,
lagging: [],
keys: ['quay'],
});
});

it('merges starts within a week into one line at the earliest', () => {
const series = (from: string) => [
{ date: from, count: 1 },
{ date: '2026-10-23', count: 1 },
];
const cov = seriesCoverage(
[
{ key: 'a', series: series('2026-09-24') },
{ key: 'b', series: series('2026-09-23') },
{ key: 'c', series: series('2026-09-28') },
],
'2026-07-26',
'2026-10-23',
);
expect(cov.starts).toEqual([{ date: '2026-09-23', keys: ['b', 'a', 'c'] }]);
});

it('treats an unfinished week as incomplete when weekly', () => {
Expand Down
87 changes: 56 additions & 31 deletions site/src/lib/summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -488,50 +488,63 @@ export function hasDataIn(
return series.some((p) => p.date >= start && p.date <= end && p.count > 0);
}

export interface Coverage {
/** Sources whose data begins inside the window, grouped by that date. */
starts: Array<{ date: string; sources: Source[] }>;
export interface Coverage<K extends string = Source> {
/**
* Series whose data begins inside the window. Starts within a week of each
* other share one line at the earliest, so a backfill that landed a day
* apart per series reads as one event.
*/
starts: Array<{ date: string; keys: K[] }>;
/** First row (day, or week when weekly) that is not complete, or null. */
partialFrom: string | null;
/** Sources that have not reported the days from `partialFrom` on. */
lagging: Source[];
/** Series that have not reported the days from `partialFrom` on. */
lagging: K[];
/** Every series with data, so a caption can say "all" instead of listing. */
keys: K[];
}

/**
* When each source's data begins and ends relative to the window, so a chart
* can mark where a band starts because collection did (not adoption) and
* which trailing days are still incomplete. When weekly, dates are week
* starts and a week that has not ended yet is incomplete too.
* When each series' data begins and ends relative to the window, so a chart
* can mark where a band starts because collection did (not usage) and which
* trailing days are still incomplete. Entries sharing a key are combined.
* When weekly, dates are week starts and a week that has not ended yet is
* incomplete too.
*/
export function coverage(
items: Countable[],
export function seriesCoverage<K extends string>(
entries: Array<{ key: K; series: Point[] }>,
start: string,
end: string,
weekly = false,
): Coverage {
const first = new Map<Source, string>();
const last = new Map<Source, string>();
for (const { source, series } of items) {
): Coverage<K> {
const first = new Map<K, string>();
const last = new Map<K, string>();
for (const { key, series } of entries) {
const a = series[0]?.date;
const b = series.at(-1)?.date;
if (!a || !b) continue;
if (a < (first.get(source) ?? '9999')) first.set(source, a);
if (b > (last.get(source) ?? '')) last.set(source, b);
if (a < (first.get(key) ?? '9999')) first.set(key, a);
if (b > (last.get(key) ?? '')) last.set(key, b);
}
const keys = [...first.keys()];
const bucket = (d: string) => (weekly ? weekStart(d) : d);
const starts = new Map<string, Source[]>();
const lagging: Source[] = [];

const starts: Coverage<K>['starts'] = [];
const late = keys
.filter((k) => (first.get(k) ?? '') > start)
.sort((x, y) => ((first.get(x) ?? '') < (first.get(y) ?? '') ? -1 : 1));
for (const key of late) {
const date = first.get(key) ?? '';
const group = starts.at(-1);
if (group && diffDays(group.date, date) <= 7) group.keys.push(key);
else starts.push({ date, keys: [key] });
}

const lagging: K[] = [];
let partial: string | null = null;
for (const source of SOURCES) {
const a = first.get(source);
const b = last.get(source);
if (!a || !b) continue;
if (a > start) {
const key = bucket(a);
starts.set(key, [...(starts.get(key) ?? []), source]);
}
for (const key of keys) {
const b = last.get(key) ?? '';
if (b < end) {
lagging.push(source);
lagging.push(key);
const next = addDays(b, 1);
if (!partial || next < partial) partial = next;
}
Expand All @@ -540,14 +553,26 @@ export function coverage(
const endsWeek = diffDays(weekStart(end), end) === 6;
if (weekly && !endsWeek && (!partial || end < partial)) partial = end;
return {
starts: [...starts]
.map(([date, sources]) => ({ date, sources }))
.sort((x, y) => (x.date < y.date ? -1 : 1)),
starts: starts.map((g) => ({ date: bucket(g.date), keys: g.keys })),
partialFrom: partial ? bucket(partial) : null,
lagging,
keys,
};
}

/** `seriesCoverage` per download source, in the dashboard's source order. */
export function coverage(
items: Countable[],
start: string,
end: string,
weekly = false,
): Coverage {
const entries = items
.map((a) => ({ key: a.source, series: a.series }))
.sort((x, y) => SOURCES.indexOf(x.key) - SOURCES.indexOf(y.key));
return seriesCoverage(entries, start, end, weekly);
}

/** Rows for a multi-series chart: one row per bucket with a key per series. */
export function chartRows(
seriesByKey: Record<string, Point[]>,
Expand Down
20 changes: 20 additions & 0 deletions site/src/pages/Docs/Docs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
import { useDashboardParams } from '@/hooks/useDashboardParams';
import { useDocs } from '@/hooks/useSummary';
import { markersFor } from '@/lib/anomalies';
import { coverageLines, partialNote } from '@/lib/coverage';
import { formatCompact, formatDate, formatNumber } from '@/lib/format';
import {
between,
Expand All @@ -30,6 +31,7 @@ import {
matchesQuery,
type Point,
RANGE_LABEL,
seriesCoverage,
windowFor,
} from '@/lib/summary';

Expand All @@ -52,6 +54,7 @@ const SITE_COLORS = [
'var(--chart-3)',
'var(--chart-4)',
'var(--chart-5)',
'var(--chart-2)',
];

function points(site: DocsSite, metric: 'visits' | 'pageviews'): Point[] {
Expand Down Expand Up @@ -104,6 +107,13 @@ function useDocsView(docs: DocsFile | undefined) {
};
});
const all = Object.values(bySite).flat();
const cov = seriesCoverage(
sites.map((site) => ({ key: site.id, series: points(site, 'visits') })),
start,
end,
weekly,
);
const siteName = (id: string) => sites.find((s) => s.id === id)?.name ?? id;
const pages = sites
.flatMap((site) =>
site.top_pages.map((p) => ({
Expand All @@ -119,6 +129,9 @@ function useDocsView(docs: DocsFile | undefined) {
.at(-1);
return {
empty: false as const,
lines: coverageLines(cov, siteName),
partialFrom: cov.partialFrom,
partialNote: partialNote(cov, weekly, siteName),
pages,
pagesDate: pagesDate || null,
asOf,
Expand Down Expand Up @@ -246,6 +259,10 @@ function Docs() {
<li>
Lines overlap rather than stack; hover for each site's number.
</li>
<li>
A dashed line marks where the data begins: Cloudflare keeps about
30 days, so history starts from Pulse's first collection.
</li>
</ul>
}
>
Expand All @@ -255,6 +272,9 @@ function Docs() {
weekly={view.weekly}
stacked={false}
markers={view.markers}
lines={view.lines}
partialFrom={view.partialFrom}
partialNote={view.partialNote}
title={`${view.weekly ? 'Weekly' : 'Daily'} docs visits by site, ${rangeLabel}`}
/>
</ChartCard>
Expand Down
3 changes: 2 additions & 1 deletion site/src/pages/Sources/Sources.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ const DOCS: Row[] = [
name: 'Visits and page views',
counts:
'HTML page views to browsers on each documentation site, at the Cloudflare edge, so cookie consent and ad blockers do not hide them. A visit is a page view not preceded by another from the same visitor in 30 minutes. Bots excluded.',
history: 'Daily; the last 30 days are refetched each run.',
history:
'Cloudflare keeps about 30 days; Pulse keeps every day since. The last 30 days are refetched each run.',
},
{
name: 'Top pages',
Expand Down
Loading