mirror of
https://github.com/jason5ng32/MyIP.git
synced 2026-11-04 08:55:47 +00:00
Refactor(layout): split backend-only code out of common/, group frontend utils
common/ had become the backend's library: 30 of its 41 modules were imported by nothing in the frontend, so the name no longer said what belonged there. It now holds only what the frontend or the build imports too; backend-only code moved to server/, flat except for datasets/ (the dataset engine and its readers). The *-db/ dataset directories stay in common/: self-hosting docs and Docker volumes mount those paths. frontend/utils/ keeps its app-wide modules at the root and gains ip/ (IP / ASN shaping and validation), report/ (the report pipeline) and features/ (the pure layer of a single feature). Paths only — no logic changes. The Dockerfile copies server/, and the AGENTS.md files say where a new module goes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jason5ng32andClaude Opus 5.5 committed
1 parent
ec4d536b60
commit
c29e48687b
209 files changed
+420
-405
No files matched your search
@@ -0,0 +1,40 @@
|
||||
// Relationship counts that agree with the /api/asn-connectivity graph. The
|
||||
// graph walk (buildTopology) drops the non-Tier-1 "providers" of an AS that
|
||||
// is adjacent to the Tier 1 clique past either bar below — CAIDA tends to
|
||||
// misinfer such links for hypergiants. These helpers apply that same rule
|
||||
// when counting, so the ASN Profile neighbour lists never show an upstream
|
||||
// the graph leaves out. Nothing is
|
||||
// reclassified: a distrusted provider is simply not counted.
|
||||
|
||||
import { providersOf, peersOf, customersOf, customerCountOf, isTier1 } from './datasets/as-rel-db.js';
|
||||
|
||||
// Peering-only bar: the hypergiant signature is clique peering (Google 12,
|
||||
// Cloudflare 7). Combined bar: multihomed hosters with a real non-T1
|
||||
// upstream sit at ≤6 Tier-1 adjacencies. buildTopology reads these too.
|
||||
export const TIER1_PEERING_TRUSTED = 5;
|
||||
export const TIER1_ADJACENCY_TRUSTED = 8;
|
||||
|
||||
// Default adjacency API; tests inject a fixture instead.
|
||||
const asRelApi = { providersOf, peersOf, customersOf, isTier1, customerCountOf };
|
||||
|
||||
const unique = (list) => [...new Set(list)];
|
||||
|
||||
// The graph's test for `asn`, verbatim: are its non-Tier-1 providers noise?
|
||||
export const distrustsNonTier1Providers = (asn, rel = asRelApi) => {
|
||||
const tier1Providers = rel.providersOf(asn).filter((p) => rel.isTier1(p));
|
||||
const tier1Peers = rel.peersOf(asn).filter((p) => rel.isTier1(p));
|
||||
return tier1Peers.length >= TIER1_PEERING_TRUSTED
|
||||
|| tier1Providers.length + tier1Peers.length >= TIER1_ADJACENCY_TRUSTED;
|
||||
};
|
||||
|
||||
// Upstreams, peers and customers of `origin` (ASN arrays). Distrusted
|
||||
// non-Tier-1 providers are left out; a pair listed as both transit and
|
||||
// peering counts as transit only.
|
||||
export const countedRelationships = (origin, rel = asRelApi) => {
|
||||
const distrust = distrustsNonTier1Providers(origin, rel);
|
||||
const providers = unique(rel.providersOf(origin)).filter((p) => !distrust || rel.isTier1(p));
|
||||
const customers = unique(rel.customersOf(origin));
|
||||
const transit = new Set([...rel.providersOf(origin), ...customers]);
|
||||
const peers = unique(rel.peersOf(origin)).filter((asn) => !transit.has(asn));
|
||||
return { providers, peers, customers };
|
||||
};
|
||||
@@ -0,0 +1,191 @@
|
||||
// Composition behind /api/asn-profile: every data source of the ASN Profile
|
||||
// page fetched in parallel, each under its own deadline, answered as one
|
||||
// object. A source that fails or runs out of time is just absent (null) with
|
||||
// its status set; the response is still whole. Pure apart from the injected
|
||||
// loaders, so tests need no network.
|
||||
//
|
||||
// Response shape (one key per section, plus `status` and `incomplete`):
|
||||
// { asn, status: { radar, prefixes, connectivity, whois, rank, reputation, peeringdb },
|
||||
// incomplete: [section, …],
|
||||
// radar, prefixes, connectivity, whois, rank, reputation, peeringdb }
|
||||
// status values: 'ok' (data present) · 'empty' (source answered, nothing
|
||||
// for this AS) · 'error' (failed / timed out, no data) · 'disabled' (not
|
||||
// configured on this deployment). Section data is non-null only for 'ok'.
|
||||
//
|
||||
// `incomplete` lists 'ok' sections whose source answered only in part (some
|
||||
// of its upstream calls failed — today Radar's segments): the data is still
|
||||
// shown, but the answer gets only the short degraded TTL (isCompleteProfile,
|
||||
// backend-server.js). A loader signals it by resolving
|
||||
// { status: 'ok', data, incomplete: true }.
|
||||
|
||||
import logger from './logger.js';
|
||||
|
||||
export const SECTIONS = ['radar', 'prefixes', 'connectivity', 'whois', 'rank', 'reputation', 'peeringdb'];
|
||||
|
||||
// Upstream timeouts api/asn-profile.js hands to the sources that take one.
|
||||
export const SOURCE_TIMEOUTS = { autnum: 5000, rank: 4000, reputation: 4000 };
|
||||
|
||||
const UPSTREAM_DEFAULT_MS = 8000; // fetchUpstream's default
|
||||
|
||||
// Worst-case time (ms) each source's own upstream timeouts allow, so a
|
||||
// source that degrades internally still answers before its deadline:
|
||||
// radar 7 parallel Radar calls at the fetchUpstream default
|
||||
// prefixes one Radar pfx2as call + local MaxMind
|
||||
// whois IANA bootstrap (default, cached 24 h), then RDAP autnum
|
||||
// connectivity local CAIDA + RIPEstat name fallback (3 s, in parallel)
|
||||
// rank / reputation ASRank GraphQL / private API
|
||||
// peeringdb local index, no I/O
|
||||
export const INNER_TIMEOUTS = {
|
||||
radar: UPSTREAM_DEFAULT_MS,
|
||||
prefixes: UPSTREAM_DEFAULT_MS,
|
||||
whois: UPSTREAM_DEFAULT_MS + SOURCE_TIMEOUTS.autnum,
|
||||
connectivity: 3000,
|
||||
rank: SOURCE_TIMEOUTS.rank,
|
||||
reputation: SOURCE_TIMEOUTS.reputation,
|
||||
peeringdb: 0,
|
||||
};
|
||||
|
||||
// Per-section budget (ms): each sits above its source's worst case
|
||||
// (INNER_TIMEOUTS), so the whole answer is bounded by the largest.
|
||||
export const DEADLINES = {
|
||||
radar: 10000,
|
||||
prefixes: 10000,
|
||||
connectivity: 5000,
|
||||
whois: 14000,
|
||||
rank: 5000,
|
||||
reputation: 5000,
|
||||
peeringdb: 1000,
|
||||
};
|
||||
|
||||
// Reject with a code 'deadline' error when `promise` outlives `ms`. The
|
||||
// underlying work keeps running; its late result is ignored.
|
||||
export const withDeadline = (promise, ms) => {
|
||||
let timer;
|
||||
const deadline = new Promise((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(Object.assign(new Error(`deadline of ${ms} ms exceeded`), { code: 'deadline' }));
|
||||
}, ms);
|
||||
});
|
||||
return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
|
||||
};
|
||||
|
||||
const result = (status, data = null) => ({ status, data: status === 'ok' ? data : null });
|
||||
|
||||
// A field worth showing in the Radar ASN summary: any name, or a figure that
|
||||
// isn't zero ('0', '0.00%', 0 — what Radar's routing stats give an AS it
|
||||
// doesn't know).
|
||||
export const hasMeaningfulField = (body) => Object.values(body || {}).some((value) => {
|
||||
if (typeof value === 'number') return Number.isFinite(value) && value !== 0;
|
||||
if (typeof value !== 'string' || !value.trim()) return false;
|
||||
const digits = value.replace(/[^\d.]/g, '');
|
||||
return !digits || parseFloat(digits) !== 0;
|
||||
});
|
||||
|
||||
// -- per-section classification of a source's answer ------------------------
|
||||
|
||||
// `answer` is loadAsnSummary's (server/cf-radar.js). With failed segments,
|
||||
// data is ok but incomplete; no data is an error, as the missing segments
|
||||
// may have held it.
|
||||
export const classifyRadar = ({ summary, failedSegments = [] } = {}) => {
|
||||
if (hasMeaningfulField(summary)) {
|
||||
return failedSegments.length > 0 ? { ...result('ok', summary), incomplete: true } : result('ok', summary);
|
||||
}
|
||||
if (failedSegments.length > 0) throw new Error(`Radar segments failed: ${failedSegments.join(', ')}`);
|
||||
return result('empty');
|
||||
};
|
||||
|
||||
export const classifyPrefixes = (body) => {
|
||||
if (!Array.isArray(body?.prefixes)) throw new Error('malformed bgp-prefixes payload');
|
||||
return result(body.prefixes.length ? 'ok' : 'empty', body);
|
||||
};
|
||||
|
||||
export const classifyConnectivity = (body) => {
|
||||
const counts = body?.neighbours?.counts || {};
|
||||
const neighbours = (counts.providers || 0) + (counts.peers || 0) + (counts.customers || 0);
|
||||
const edges = Array.isArray(body?.edges) ? body.edges.length : 0;
|
||||
return result(edges || neighbours ? 'ok' : 'empty', body);
|
||||
};
|
||||
|
||||
export const classifyRank = (record) => result(record?.rank != null ? 'ok' : 'empty', record);
|
||||
|
||||
// `answer` is requestAsnReputation's: null = not configured. Only a 200
|
||||
// with a boolean `found` is an answer; any other status or shape throws.
|
||||
export const classifyReputation = (answer) => {
|
||||
if (!answer) return result('disabled');
|
||||
if (answer.status !== 200) throw new Error(`reputation upstream responded ${answer.status}`);
|
||||
if (typeof answer.data?.found !== 'boolean') throw new Error('malformed reputation payload');
|
||||
return result(answer.data.found ? 'ok' : 'empty', answer.data);
|
||||
};
|
||||
|
||||
// `loaded` false = no PeeringDB index on this deployment; `record` is
|
||||
// lookupPeeringdb's (null when the AS has none).
|
||||
export const classifyPeeringdb = (loaded, record) => {
|
||||
if (!loaded) return result('disabled');
|
||||
return result(record ? 'ok' : 'empty', record);
|
||||
};
|
||||
|
||||
// -- loaders ------------------------------------------------------------------
|
||||
|
||||
// Section loaders over the real data functions (injected — see
|
||||
// api/asn-profile.js). Each resolves { status, data, incomplete? } or throws.
|
||||
// deps: { hasRadarKey(), fetchRadarAsn(asn) (loadAsnSummary's shape),
|
||||
// fetchRadarPrefixes(asn), getConnectivity(asn), rdapAutnum(asn),
|
||||
// isAutnumMissing(err), queryAsRank(asn), requestReputation(asn),
|
||||
// isPeeringdbLoaded(), lookupPeeringdb(asn),
|
||||
// isPartialPrefixes(body)? — true marks an 'ok' prefixes section incomplete }
|
||||
export const buildSectionLoaders = (deps) => ({
|
||||
radar: async (asn) => (deps.hasRadarKey() ? classifyRadar(await deps.fetchRadarAsn(asn)) : result('disabled')),
|
||||
prefixes: async (asn) => {
|
||||
if (!deps.hasRadarKey()) return result('disabled');
|
||||
const body = await deps.fetchRadarPrefixes(asn);
|
||||
const section = classifyPrefixes(body);
|
||||
// e.g. countries missing for want of MaxMind: shown, not cached for a week.
|
||||
return section.status === 'ok' && deps.isPartialPrefixes?.(body) ? { ...section, incomplete: true } : section;
|
||||
},
|
||||
connectivity: async (asn) => classifyConnectivity(await deps.getConnectivity(asn)),
|
||||
whois: async (asn) => {
|
||||
try {
|
||||
return result('ok', await deps.rdapAutnum(asn));
|
||||
} catch (error) {
|
||||
if (deps.isAutnumMissing(error)) return result('empty');
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
rank: async (asn) => classifyRank(await deps.queryAsRank(asn)),
|
||||
reputation: async (asn) => classifyReputation(await deps.requestReputation(asn)),
|
||||
peeringdb: async (asn) => classifyPeeringdb(deps.isPeeringdbLoaded(), deps.lookupPeeringdb(asn)),
|
||||
});
|
||||
|
||||
// Run every loader under its deadline and assemble the response body.
|
||||
export const composeAsnProfile = async (asn, loaders, { deadlines = DEADLINES } = {}) => {
|
||||
const settled = await Promise.allSettled(
|
||||
SECTIONS.map((section) => withDeadline(Promise.resolve().then(() => loaders[section](asn)), deadlines[section])),
|
||||
);
|
||||
const body = { asn, status: {}, incomplete: [] };
|
||||
settled.forEach((outcome, i) => {
|
||||
const section = SECTIONS[i];
|
||||
if (outcome.status === 'fulfilled') {
|
||||
body.status[section] = outcome.value.status;
|
||||
body[section] = outcome.value.data;
|
||||
if (outcome.value.incomplete && outcome.value.status === 'ok') body.incomplete.push(section);
|
||||
} else {
|
||||
logger.warn({ err: outcome.reason, asn, section }, 'asn-profile: section failed');
|
||||
body.status[section] = 'error';
|
||||
body[section] = null;
|
||||
}
|
||||
});
|
||||
return body;
|
||||
};
|
||||
|
||||
// Every configured source failed — the only case answered as an error.
|
||||
export const allSourcesFailed = (body) => {
|
||||
const live = Object.values(body?.status || {}).filter((status) => status !== 'disabled');
|
||||
return live.length > 0 && live.every((status) => status === 'error');
|
||||
};
|
||||
|
||||
// Whether an answer earns the full edge-cache TTL: no section in error or
|
||||
// incomplete. A degraded one gets a day (backend-server.js), so a failure is
|
||||
// never pinned for a week.
|
||||
export const isCompleteProfile = (body) => Boolean(body?.status)
|
||||
&& !Object.values(body.status).includes('error')
|
||||
&& !(body.incomplete?.length > 0);
|
||||
@@ -0,0 +1,23 @@
|
||||
// ASN reputation from the main IPCheck.ing API
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
|
||||
// Resolves null when this deployment has no private API configured, else
|
||||
// { status, data } as the upstream answered (an unknown ASN is a 200 with
|
||||
// found: false, a table still loading is a 503). A non-2xx body that isn't
|
||||
// JSON comes back as data: null; a 2xx body that isn't JSON rejects, like a
|
||||
// network failure — neither may pass for "no reputation".
|
||||
export const requestAsnReputation = async (asn, headers, { timeoutMs } = {}) => {
|
||||
const apiKey = process.env.IPCHECKING_API_KEY;
|
||||
const apiEndpoint = process.env.IPCHECKING_API_ENDPOINT;
|
||||
if (!apiKey || !apiEndpoint) return null;
|
||||
|
||||
const url = new URL(`${apiEndpoint}/asnreputation`);
|
||||
url.searchParams.set('asn', asn);
|
||||
url.searchParams.set('key', apiKey);
|
||||
const apiResponse = await fetchUpstream(url, { headers: { ...headers }, ...(timeoutMs && { timeoutMs }) });
|
||||
const data = apiResponse.ok
|
||||
? await apiResponse.json()
|
||||
: await apiResponse.json().catch(() => null);
|
||||
return { status: apiResponse.status, data };
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
// CAIDA ASRank client — global rank, customer cone and degree for one ASN via
|
||||
// their GraphQL API, for /api/asn-profile. Academic service with no SLA:
|
||||
// queryAsRank throws on any upstream failure, and the aggregate marks the
|
||||
// section failed (and keeps that answer off the edge cache).
|
||||
//
|
||||
// ASRank's customer cone is path-observed and is the citable number; a naive
|
||||
// transitive closure over as-rel p2c edges overestimates it badly for
|
||||
// mixed-role networks.
|
||||
//
|
||||
// Source: https://api.asrank.caida.org/v2/graphql (free, no auth)
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
|
||||
const ASRANK_ENDPOINT = 'https://api.asrank.caida.org/v2/graphql';
|
||||
|
||||
const QUERY = `query Asn($asn: String!) {
|
||||
asn(asn: $asn) {
|
||||
rank
|
||||
asnName
|
||||
organization { orgName }
|
||||
country { iso }
|
||||
asnDegree { provider peer customer transit }
|
||||
cone { numberAsns numberPrefixes numberAddresses }
|
||||
}
|
||||
}`;
|
||||
|
||||
// GraphQL response body → flat record; null when the ASN is unknown to
|
||||
// ASRank. Exported for tests.
|
||||
export const mapAsRankResponse = (body) => {
|
||||
const asn = body?.data?.asn;
|
||||
if (!asn) return null;
|
||||
return {
|
||||
rank: asn.rank ?? null,
|
||||
asnName: asn.asnName || null,
|
||||
orgName: asn.organization?.orgName || null,
|
||||
country: asn.country?.iso || null,
|
||||
degree: {
|
||||
providers: asn.asnDegree?.provider ?? null,
|
||||
peers: asn.asnDegree?.peer ?? null,
|
||||
customers: asn.asnDegree?.customer ?? null,
|
||||
transits: asn.asnDegree?.transit ?? null,
|
||||
},
|
||||
cone: {
|
||||
asns: asn.cone?.numberAsns ?? null,
|
||||
prefixes: asn.cone?.numberPrefixes ?? null,
|
||||
addresses: asn.cone?.numberAddresses ?? null,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
// A 200 body is only an answer once GraphQL says so: any `errors` entry
|
||||
// fails it, even beside a usable `data.asn` — fields a failed resolver left
|
||||
// null would read as "ASRank has no figure" and get cached. A body without a
|
||||
// `data` object is malformed. Exported for tests.
|
||||
export const checkAsRankBody = (body) => {
|
||||
if (Array.isArray(body?.errors) && body.errors.length > 0) {
|
||||
throw new Error(`ASRank GraphQL error: ${body.errors[0]?.message || 'unknown'}`);
|
||||
}
|
||||
if (!body?.data || typeof body.data !== 'object') throw new Error('ASRank returned a malformed body');
|
||||
return body;
|
||||
};
|
||||
|
||||
// ASRank record for an ASN; null when ASRank doesn't know it. Throws on an
|
||||
// upstream failure (non-2xx, unparsable body, GraphQL errors), so callers
|
||||
// that need to tell the two apart can.
|
||||
export const queryAsRank = async (asn, { timeoutMs = 4000 } = {}) => {
|
||||
const res = await fetchUpstream(ASRANK_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ query: QUERY, variables: { asn: String(asn) } }),
|
||||
timeoutMs,
|
||||
});
|
||||
if (!res.ok) throw new Error(`ASRank responded ${res.status}`);
|
||||
return mapAsRankResponse(checkAsRankBody(await res.json()));
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
// Edge-cache middleware for /api routes (attached in backend-server.js).
|
||||
// Every /api response defaults to no-store; a route wrapped in
|
||||
// cacheable(maxAge) gets `public, max-age=…` on its 2xx JSON answers only —
|
||||
// the edge must never cache a 4xx / 5xx.
|
||||
//
|
||||
// maxAge seconds, or a `(req) => seconds` resolver for routes whose
|
||||
// TTL depends on the request (the /api/cfradar views); a
|
||||
// falsy resolution keeps the no-store default
|
||||
// cacheIf(body) optional veto for a 2xx body that answers a degraded
|
||||
// upstream (an /api/asn-profile section in error)
|
||||
// degradedMaxAge optional TTL for a body cacheIf vetoes, as a number or a
|
||||
// `() => seconds` resolver read when the body is sent;
|
||||
// falsy = no-store, the default
|
||||
//
|
||||
// The full-TTL header is also stashed on res.locals.cacheControl, so
|
||||
// binary-stream handlers (which bypass res.json) can apply it on their own
|
||||
// 2xx path.
|
||||
|
||||
const resolve = (value, ...args) => (typeof value === 'function' ? value(...args) : value);
|
||||
|
||||
export const cacheable = (maxAge, { cacheIf, degradedMaxAge } = {}) => (req, res, next) => {
|
||||
const maxAgeSeconds = resolve(maxAge, req);
|
||||
if (maxAgeSeconds) {
|
||||
res.locals.cacheControl = `public, max-age=${maxAgeSeconds}`;
|
||||
const originalJson = res.json.bind(res);
|
||||
res.json = (body) => {
|
||||
if (res.statusCode < 400) {
|
||||
if (!cacheIf || cacheIf(body)) {
|
||||
res.setHeader('Cache-Control', res.locals.cacheControl);
|
||||
} else {
|
||||
const degradedSeconds = resolve(degradedMaxAge);
|
||||
if (degradedSeconds) res.setHeader('Cache-Control', `public, max-age=${degradedSeconds}`);
|
||||
}
|
||||
}
|
||||
return originalJson(body);
|
||||
};
|
||||
}
|
||||
next();
|
||||
};
|
||||
@@ -0,0 +1,445 @@
|
||||
// Cloudflare Radar data layer behind the single /api/cfradar route. Each
|
||||
// RADAR_VIEWS entry is one `?view=` value: the guard middlewares that
|
||||
// validate/normalize its query params (reused from server/guards.js), the
|
||||
// edge-cache TTL the route middleware in backend-server.js applies, and the
|
||||
// fetch function that produces the response payload. api/cf-radar.js
|
||||
// dispatches over this registry — adding Radar data means one view function
|
||||
// plus one registry row here, never a new route.
|
||||
//
|
||||
// A view may answer despite failed upstream calls (asn: some segments) or
|
||||
// with local lookups left undone (bgp-prefixes: countries MaxMind couldn't
|
||||
// answer). It then
|
||||
// registers the payload via markPartial; the route's cache middleware asks
|
||||
// isCompleteRadarAnswer and serves such an answer without edge-caching it.
|
||||
// The payload itself is unchanged.
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
import { requireValidASN, requireValidCountry } from './guards.js';
|
||||
import { lookupCountryRange } from './datasets/maxmind-service.js';
|
||||
import { parseCidr, formatIPv4 } from '../common/ip-math.js';
|
||||
import logger from './logger.js';
|
||||
|
||||
// -- shared Radar client ----------------------------------------------------
|
||||
|
||||
// CLOUDFLARE_API is the pre-rename spelling — keep reading it so existing
|
||||
// deployments don't lose the key on upgrade.
|
||||
export const hasRadarApiKey = () =>
|
||||
Boolean(process.env.CLOUDFLARE_API_KEY || process.env.CLOUDFLARE_API);
|
||||
|
||||
export async function fetchFromCloudflare(endpoint) {
|
||||
const url = `https://api.cloudflare.com/client/v4${endpoint}`;
|
||||
const response = await fetchUpstream(url, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${process.env.CLOUDFLARE_API_KEY || process.env.CLOUDFLARE_API}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
// Outage pages come back as HTML — fail on status instead of JSON.parse.
|
||||
if (!response.ok) {
|
||||
throw Object.assign(new Error(`Cloudflare Radar responded ${response.status}`), { status: response.status });
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// Payloads served despite failed upstream calls (see the header). Keyed by
|
||||
// object identity, so nothing leaks into the JSON.
|
||||
const partialAnswers = new WeakSet();
|
||||
const markPartial = (body) => {
|
||||
partialAnswers.add(body);
|
||||
return body;
|
||||
};
|
||||
export const isCompleteRadarAnswer = (body) => !partialAnswers.has(body);
|
||||
|
||||
// -- view: asn — per-ASN entity info + 7d HTTP traffic profile --------------
|
||||
|
||||
// The Radar segments backing one response, keyed by the field name
|
||||
// cleanUpResponseData expects. Prefix and relationship counts live in the
|
||||
// ASN Profile (pfx2as list, CAIDA graph), not here.
|
||||
const SEGMENTS = {
|
||||
asnInfo: (asn) => `/radar/entities/asns/${asn}`,
|
||||
ipVersion: (asn) => `/radar/http/summary/ip_version?asn=${asn}&dateRange=7d`,
|
||||
httpProtocol: (asn) => `/radar/http/summary/http_protocol?asn=${asn}&dateRange=7d`,
|
||||
deviceType: (asn) => `/radar/http/summary/device_type?asn=${asn}&dateRange=7d`,
|
||||
botType: (asn) => `/radar/http/summary/bot_class?asn=${asn}&dateRange=7d`,
|
||||
quality: (asn) => `/radar/quality/speed/summary?asn=${asn}`,
|
||||
};
|
||||
|
||||
// Fetch all segments in parallel. A failed segment is dropped rather than
|
||||
// failing the whole response — cleanUpResponseData / filterData already
|
||||
// tolerate sparse data (small ASNs), so partial results degrade to missing
|
||||
// fields. Only a full wipe-out is an error (loadAsnSummary throws).
|
||||
const getAllASNData = async (asn) => {
|
||||
const names = Object.keys(SEGMENTS);
|
||||
const settled = await Promise.allSettled(names.map((name) => fetchFromCloudflare(SEGMENTS[name](asn))));
|
||||
const data = {};
|
||||
const failed = [];
|
||||
settled.forEach((result, i) => {
|
||||
if (result.status === 'fulfilled') {
|
||||
data[names[i]] = result.value;
|
||||
} else if (result.reason?.status !== 404) {
|
||||
// A 404 is Radar's "no such AS" — an empty answer, not a fault.
|
||||
failed.push({ name: names[i], reason: result.reason });
|
||||
}
|
||||
});
|
||||
return { data, failed };
|
||||
};
|
||||
|
||||
// Clean up Cloudflare Radar return data to uniform field names.
|
||||
// Optional-chaining everywhere because CF Radar returns sparse data for
|
||||
// small / private / new ASNs (e.g. AS64512 is in the RFC 6996 private range
|
||||
// and has no info at all; many smaller ASNs have asn info but no traffic
|
||||
// summaries). Missing fields fall through as undefined and get stripped
|
||||
// downstream in filterData via the NaN check.
|
||||
function cleanUpResponseData(data) {
|
||||
return {
|
||||
asnName: data.asnInfo?.result?.asn?.name,
|
||||
asnCountryCode: data.asnInfo?.result?.asn?.country,
|
||||
asnOrgName: data.asnInfo?.result?.asn?.orgName,
|
||||
estimatedUsers: data.asnInfo?.result?.asn?.estimatedUsers?.estimatedUsers,
|
||||
IPv4_Pct: data.ipVersion?.result?.summary_0?.IPv4,
|
||||
IPv6_Pct: data.ipVersion?.result?.summary_0?.IPv6,
|
||||
HTTP_Pct: data.httpProtocol?.result?.summary_0?.http,
|
||||
HTTPS_Pct: data.httpProtocol?.result?.summary_0?.https,
|
||||
Desktop_Pct: data.deviceType?.result?.summary_0?.desktop,
|
||||
Mobile_Pct: data.deviceType?.result?.summary_0?.mobile,
|
||||
Bot_Pct: data.botType?.result?.summary_0?.bot,
|
||||
Human_Pct: data.botType?.result?.summary_0?.human,
|
||||
speedDownload: data.quality?.result?.summary_0?.bandwidthDownload,
|
||||
speedUpload: data.quality?.result?.summary_0?.bandwidthUpload,
|
||||
latency: data.quality?.result?.summary_0?.latencyIdle,
|
||||
jitter: data.quality?.result?.summary_0?.jitterIdle,
|
||||
};
|
||||
}
|
||||
|
||||
// Format output; a missing value parses to NaN, which filterData drops.
|
||||
function formatData(data) {
|
||||
const { asnName, asnCountryCode, asnOrgName, estimatedUsers, IPv4_Pct, IPv6_Pct, HTTP_Pct, HTTPS_Pct, Desktop_Pct, Mobile_Pct, Bot_Pct, Human_Pct, speedDownload, speedUpload, latency, jitter } = data;
|
||||
return {
|
||||
asnName,
|
||||
asnCountryCode,
|
||||
asnOrgName,
|
||||
estimatedUsers: parseFloat(estimatedUsers).toLocaleString(),
|
||||
speedDownload: `${parseFloat(speedDownload).toFixed(1)} Mbps`,
|
||||
speedUpload: `${parseFloat(speedUpload).toFixed(1)} Mbps`,
|
||||
latency: `${Math.round(parseFloat(latency))} ms`,
|
||||
jitter: `${parseFloat(jitter).toFixed(1)} ms`,
|
||||
IPv4_Pct: `${parseFloat(IPv4_Pct).toFixed(2)}%`,
|
||||
IPv6_Pct: `${parseFloat(IPv6_Pct).toFixed(2)}%`,
|
||||
HTTP_Pct: `${parseFloat(HTTP_Pct).toFixed(2)}%`,
|
||||
HTTPS_Pct: `${parseFloat(HTTPS_Pct).toFixed(2)}%`,
|
||||
Desktop_Pct: `${parseFloat(Desktop_Pct).toFixed(2)}%`,
|
||||
Mobile_Pct: `${parseFloat(Mobile_Pct).toFixed(2)}%`,
|
||||
Bot_Pct: `${parseFloat(Bot_Pct).toFixed(2)}%`,
|
||||
Human_Pct: `${parseFloat(Human_Pct).toFixed(2)}%`,
|
||||
};
|
||||
}
|
||||
|
||||
// Filter out non-existent fields — a missing upstream value formats into a
|
||||
// string leading with "NaN" whatever unit suffix it carries.
|
||||
function filterData(data) {
|
||||
for (const key in data) {
|
||||
if (String(data[key]).startsWith('NaN')) {
|
||||
delete data[key];
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// Settled segment payloads → response body. Exported for tests.
|
||||
export const shapeAsnProfile = (data) => filterData(formatData(cleanUpResponseData(data)));
|
||||
|
||||
// One AS's summary, for the asn view and /api/asn-profile:
|
||||
// summary the view's payload
|
||||
// failedSegments names of the segments that failed; [] = complete
|
||||
// Throws when every segment failed.
|
||||
export const loadAsnSummary = async (asn) => {
|
||||
const { data, failed } = await getAllASNData(asn);
|
||||
if (failed.length === Object.keys(SEGMENTS).length) {
|
||||
throw failed[0].reason instanceof Error
|
||||
? failed[0].reason
|
||||
: new Error('all Radar segments failed');
|
||||
}
|
||||
if (failed.length > 0) {
|
||||
logger.warn({ err: failed[0].reason, asn, segments: failed.map((f) => f.name) }, 'cf-radar: partial Radar segment failure');
|
||||
}
|
||||
return { summary: shapeAsnProfile(data), failedSegments: failed.map((f) => f.name) };
|
||||
};
|
||||
|
||||
// Partial summaries are served but not edge-cached.
|
||||
const fetchAsnProfile = async ({ asn }) => {
|
||||
const { summary, failedSegments } = await loadAsnSummary(asn);
|
||||
return failedSegments.length > 0 ? markPartial(summary) : summary;
|
||||
};
|
||||
|
||||
// -- view: bgp-prefixes — announced prefixes + IPv4 country footprint -------
|
||||
|
||||
// Per-prefix origination list from Radar's pfx2as. One row per announced
|
||||
// prefix (v4/v6 mixed): rpki passes through Radar's validation verdict
|
||||
// (Valid / Invalid / Unknown), peers is how many route collectors carry the
|
||||
// route. A result without a `prefix_origins` array throws: only a real empty
|
||||
// list means "originates nothing". Exported for tests.
|
||||
export const normalizePrefixOrigins = (result) => {
|
||||
if (!Array.isArray(result?.prefix_origins)) throw new Error('malformed pfx2as payload');
|
||||
return result.prefix_origins.map((row) => ({
|
||||
prefix: row.prefix,
|
||||
rpki: row.rpki_validation,
|
||||
peers: row.peer_count ?? 0,
|
||||
}));
|
||||
};
|
||||
|
||||
const GEO_TOP = 50; // countries in the response
|
||||
const GEO_PREFIX_CAP = 20000; // bound on prefixes considered per request
|
||||
const GEO_LOOKUP_BUDGET = 100000; // hard cap on MaxMind lookups per request (~0.3 s)
|
||||
|
||||
// Share of the ASN's announced IPv4 addresses per geolocated country, largest
|
||||
// first. v4-only and address-weighted (a /16 counts 256× a /24); prefixes
|
||||
// covered by another announced prefix are skipped so a covering route plus
|
||||
// its more-specifics isn't counted twice. Each prefix is walked one MaxMind
|
||||
// network at a time, so one spanning several countries is split among them.
|
||||
// Shares are of the addresses that resolved to a country.
|
||||
// `lookupRange(ip) → { country, prefixLength } | null` is injected so tests
|
||||
// need no MaxMind database; null = no answer (no database, or the lookup
|
||||
// failed), unlike an address MaxMind places nowhere ({ country: null }).
|
||||
// Resolves { shares, complete }: a null answer or the budget ending the walk
|
||||
// gives { shares: [], complete: false } — shares over part of the space
|
||||
// would read as the whole distribution — and the caller must not cache it.
|
||||
// Exported for tests.
|
||||
export const buildCountryShares = (prefixes, lookupRange) => {
|
||||
const blocks = (prefixes || [])
|
||||
.map((row) => parseCidr(row?.prefix))
|
||||
.filter((cidr) => cidr?.family === 4)
|
||||
.slice(0, GEO_PREFIX_CAP)
|
||||
.map((cidr) => ({ start: cidr.network, size: 2n ** BigInt(32 - cidr.prefix) }))
|
||||
.sort((a, b) => (a.start === b.start ? Number(b.size - a.size) : (a.start < b.start ? -1 : 1)));
|
||||
|
||||
const weightByCountry = new Map();
|
||||
let total = 0;
|
||||
let coveredUntil = -1n;
|
||||
let budget = GEO_LOOKUP_BUDGET;
|
||||
for (const { start, size } of blocks) {
|
||||
// CIDR blocks nest or are disjoint: starting inside the last kept
|
||||
// block means being inside it.
|
||||
if (start <= coveredUntil) continue;
|
||||
coveredUntil = start + size - 1n;
|
||||
// IPv4 fits a Number. Each lookup answers for the aligned network
|
||||
// holding the cursor.
|
||||
const stop = Number(start + size);
|
||||
let cursor = Number(start);
|
||||
while (cursor < stop) {
|
||||
if (budget-- <= 0) return { shares: [], complete: false };
|
||||
const hit = lookupRange(formatIPv4(BigInt(cursor)));
|
||||
if (!hit) return { shares: [], complete: false };
|
||||
const networkSize = 2 ** (32 - hit.prefixLength);
|
||||
const next = (Math.floor(cursor / networkSize) + 1) * networkSize;
|
||||
const weight = Math.min(next, stop) - cursor;
|
||||
if (hit.country) {
|
||||
weightByCountry.set(hit.country, (weightByCountry.get(hit.country) || 0) + weight);
|
||||
total += weight;
|
||||
}
|
||||
cursor = next;
|
||||
}
|
||||
}
|
||||
if (!total) return { shares: [], complete: true };
|
||||
const shares = [...weightByCountry.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, GEO_TOP)
|
||||
.map(([country, weight]) => ({ country, share: Math.round((weight / total) * 10000) / 10000 }))
|
||||
.filter((row) => row.share > 0);
|
||||
return { shares, complete: true };
|
||||
};
|
||||
|
||||
// `lookupRange` is injectable for tests.
|
||||
const fetchBgpPrefixes = async ({ asn }, { lookupRange = lookupCountryRange } = {}) => {
|
||||
const json = await fetchFromCloudflare(`/radar/bgp/routes/pfx2as?origin=${asn}`);
|
||||
// An ASN originating nothing is a valid, cacheable answer: both lists
|
||||
// empty. A malformed result throws (500 here, `error` in the profile).
|
||||
const prefixes = normalizePrefixOrigins(json?.result);
|
||||
const { shares, complete } = buildCountryShares(prefixes, lookupRange);
|
||||
// Countries MaxMind couldn't fully work out (no database, a failed
|
||||
// lookup, the budget) are served, never cached. No IPv4 prefixes means
|
||||
// no lookups: complete, and cacheable.
|
||||
const body = { prefixes, countries: shares };
|
||||
return complete ? body : markPartial(body);
|
||||
};
|
||||
|
||||
// -- view: country-traffic — country online-activity heatmap ----------------
|
||||
|
||||
// Aggregates Radar's hourly HTTP-requests timeseries (requests track people
|
||||
// being online better than NetFlows bytes) into a Monday-first 7×24
|
||||
// weekday/hour matrix. Country-level on purpose: global ASNs have no single
|
||||
// diurnal rhythm, and ~250 cache keys let the 30-day edge cache absorb
|
||||
// nearly every request.
|
||||
//
|
||||
// 28d × 1h ≈ 4 samples per weekday/hour cell — one week alone would let a
|
||||
// single holiday skew the heatmap.
|
||||
const seriesEndpoint = (country, humanOnly) =>
|
||||
`/radar/http/timeseries?location=${country}${humanOnly ? '&botClass=LIKELY_HUMAN' : ''}&dateRange=28d&aggInterval=1h`;
|
||||
|
||||
// Mean of Radar's MIN0_MAX-normalized values per weekday/hour cell, re-scaled
|
||||
// so the busiest cell is 1. Stays in UTC — the response is edge-cached across
|
||||
// timezones, so the frontend does the hour rotation. Null when the series
|
||||
// covers less than a full week of hourly points. Exported for tests.
|
||||
export const buildTrafficMatrix = (serie) => {
|
||||
const timestamps = serie?.timestamps;
|
||||
const values = serie?.values;
|
||||
if (!Array.isArray(timestamps) || !Array.isArray(values)) return null;
|
||||
|
||||
const sums = Array.from({ length: 7 }, () => new Array(24).fill(0));
|
||||
const counts = Array.from({ length: 7 }, () => new Array(24).fill(0));
|
||||
let usable = 0;
|
||||
timestamps.forEach((ts, i) => {
|
||||
const value = parseFloat(values[i]);
|
||||
const ms = Date.parse(ts);
|
||||
if (!Number.isFinite(value) || !Number.isFinite(ms)) return;
|
||||
const date = new Date(ms);
|
||||
const day = (date.getUTCDay() + 6) % 7; // getUTCDay is Sunday-first
|
||||
sums[day][date.getUTCHours()] += value;
|
||||
counts[day][date.getUTCHours()] += 1;
|
||||
usable += 1;
|
||||
});
|
||||
if (usable < 7 * 24) return null;
|
||||
|
||||
const means = sums.map((row, d) => row.map((sum, h) => (counts[d][h] ? sum / counts[d][h] : 0)));
|
||||
const max = Math.max(...means.flat());
|
||||
if (max <= 0) return null;
|
||||
return means.map((row) => row.map((v) => Math.round((v / max) * 1000) / 1000));
|
||||
};
|
||||
|
||||
const fetchCountryTraffic = async (query) => {
|
||||
// Strict '1' keeps the cache key space to two variants per country.
|
||||
const humanOnly = query.human === '1';
|
||||
const json = await fetchFromCloudflare(seriesEndpoint(query.country, humanOnly));
|
||||
// null (no usable series) is a valid, cacheable answer.
|
||||
return { trafficMatrix: buildTrafficMatrix(json?.result?.serie_0) };
|
||||
};
|
||||
|
||||
// -- view: outages — global internet outage feed for the Pulse panel -------
|
||||
|
||||
// Merges two Radar sources into one recent-events list:
|
||||
// - /radar/annotations/outages (manually verified outages, with a cause)
|
||||
// - /radar/traffic_anomalies (auto-detected drops, all statuses)
|
||||
// A verified anomaly is often promoted to an outage entry for the same event,
|
||||
// so anomalies that match an outage's subject and start time are dropped.
|
||||
|
||||
const DATE_RANGE = '30d';
|
||||
const MAX_EVENTS = 30;
|
||||
// An anomaly counts as a duplicate of an outage when they share a subject
|
||||
// (location or ASN) and start within this window of each other.
|
||||
const DEDUPE_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
// Both kinds normalize to one flat shape; anomaly-only fields stay null on
|
||||
// outages and vice versa. `endDate: null` means the event is still ongoing.
|
||||
// Exported for tests.
|
||||
export const normalizeOutages = (annotations) => annotations.map((entry) => ({
|
||||
kind: 'outage',
|
||||
id: `o-${entry.id}`,
|
||||
startDate: entry.startDate,
|
||||
endDate: entry.endDate || null,
|
||||
locations: (entry.locationsDetails || []).map((loc) => loc.code),
|
||||
asns: (entry.asnsDetails || []).map((detail) => ({ asn: Number(detail.asn), name: detail.name })),
|
||||
cause: entry.outage?.outageCause || 'UNKNOWN',
|
||||
level: entry.outage?.outageType || null,
|
||||
description: entry.description || null,
|
||||
scope: entry.scope || null,
|
||||
linkedUrl: entry.linkedUrl || null,
|
||||
}));
|
||||
|
||||
// Anomaly statuses hidden from the feed. Live data carries values beyond the
|
||||
// documented VERIFIED/UNVERIFIED pair (e.g. TP = third-party confirmed), so
|
||||
// this is a blocklist, not an allowlist. Empty today — every status ships;
|
||||
// add 'UNVERIFIED' here to restrict the feed to confirmed events only.
|
||||
const EXCLUDED_ANOMALY_STATUSES = new Set([]);
|
||||
|
||||
export const normalizeAnomalies = (anomalies) => anomalies
|
||||
.filter((entry) => !EXCLUDED_ANOMALY_STATUSES.has(entry.status))
|
||||
.map((entry) => ({
|
||||
kind: 'anomaly',
|
||||
id: `a-${entry.uuid}`,
|
||||
startDate: entry.startDate,
|
||||
endDate: entry.endDate || null,
|
||||
locations: entry.locationDetails?.code ? [entry.locationDetails.code] : [],
|
||||
asns: entry.asnDetails?.asn
|
||||
? [{ asn: Number(entry.asnDetails.asn), name: entry.asnDetails.name }]
|
||||
: [],
|
||||
cause: null,
|
||||
level: null,
|
||||
description: null,
|
||||
scope: null,
|
||||
linkedUrl: null,
|
||||
}));
|
||||
|
||||
export const mergeEvents = (outages, anomalies) => {
|
||||
const isDuplicate = (anomaly) => outages.some((outage) => {
|
||||
const sharesSubject = anomaly.locations.some((code) => outage.locations.includes(code))
|
||||
|| anomaly.asns.some((a) => outage.asns.some((o) => o.asn === a.asn));
|
||||
if (!sharesSubject) return false;
|
||||
return Math.abs(Date.parse(anomaly.startDate) - Date.parse(outage.startDate)) <= DEDUPE_WINDOW_MS;
|
||||
});
|
||||
// Two-level order, consumed by the frontend as-is: ongoing events first
|
||||
// (endDate null), ended ones after, newest-first inside each group. The
|
||||
// cap then favors ongoing events by construction.
|
||||
return [...outages, ...anomalies.filter((anomaly) => !isDuplicate(anomaly))]
|
||||
.sort((a, b) => {
|
||||
const aOngoing = a.endDate === null;
|
||||
const bOngoing = b.endDate === null;
|
||||
if (aOngoing !== bOngoing) return aOngoing ? -1 : 1;
|
||||
return Date.parse(b.startDate) - Date.parse(a.startDate);
|
||||
})
|
||||
.slice(0, MAX_EVENTS);
|
||||
};
|
||||
|
||||
const fetchOutages = async () => {
|
||||
// One failed source degrades to a partial feed; both failing is an error.
|
||||
const [outagesResult, anomaliesResult] = await Promise.allSettled([
|
||||
fetchFromCloudflare(`/radar/annotations/outages?dateRange=${DATE_RANGE}&limit=50`),
|
||||
fetchFromCloudflare(`/radar/traffic_anomalies?dateRange=${DATE_RANGE}&limit=100`),
|
||||
]);
|
||||
|
||||
if (outagesResult.status === 'rejected' && anomaliesResult.status === 'rejected') {
|
||||
throw outagesResult.reason instanceof Error
|
||||
? outagesResult.reason
|
||||
: new Error('both Radar sources failed');
|
||||
}
|
||||
if (outagesResult.status === 'rejected' || anomaliesResult.status === 'rejected') {
|
||||
const failed = outagesResult.status === 'rejected' ? outagesResult : anomaliesResult;
|
||||
logger.warn({ err: failed.reason }, 'cf-radar: one outage source failed');
|
||||
}
|
||||
|
||||
const outages = normalizeOutages(
|
||||
outagesResult.status === 'fulfilled' ? outagesResult.value?.result?.annotations || [] : []);
|
||||
const anomalies = normalizeAnomalies(
|
||||
anomaliesResult.status === 'fulfilled' ? anomaliesResult.value?.result?.trafficAnomalies || [] : []);
|
||||
|
||||
return { events: mergeEvents(outages, anomalies) };
|
||||
};
|
||||
|
||||
// -- view registry ----------------------------------------------------------
|
||||
|
||||
// guards: run by the dispatcher before fetch — each 400s invalid input and
|
||||
// normalizes params in place, so cache keys stay canonical.
|
||||
// ttl: edge-cache seconds, read by the /api/cfradar route middleware.
|
||||
// fetch: async (req.query) => payload; throws on upstream failure.
|
||||
export const RADAR_VIEWS = {
|
||||
// The ASN data family (this, bgp-prefixes, /api/asn-profile) shares one
|
||||
// week of edge cache: the profile doesn't chase day-level precision.
|
||||
'asn': {
|
||||
guards: [requireValidASN()],
|
||||
ttl: 7 * 24 * 60 * 60,
|
||||
fetch: fetchAsnProfile,
|
||||
},
|
||||
'country-traffic': {
|
||||
guards: [requireValidCountry()],
|
||||
ttl: 30 * 24 * 60 * 60,
|
||||
fetch: fetchCountryTraffic,
|
||||
},
|
||||
'outages': {
|
||||
guards: [],
|
||||
ttl: 60 * 60,
|
||||
fetch: fetchOutages,
|
||||
},
|
||||
'bgp-prefixes': {
|
||||
guards: [requireValidASN()],
|
||||
ttl: 7 * 24 * 60 * 60,
|
||||
fetch: fetchBgpPrefixes,
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,103 @@
|
||||
// Local CAIDA as2org lookup. Snapshot file is auto-downloaded by the
|
||||
// dataset updater (server/datasets/datasets.js); this module just parses and serves.
|
||||
//
|
||||
// We use CAIDA's pipe-delimited TXT (~12MB) rather than the equivalent
|
||||
// JSONL (~28MB): identical content but split('|') beats JSON.parse per
|
||||
// line by ~40%.
|
||||
//
|
||||
// Source: https://publicdata.caida.org/datasets/as-organizations/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const AS_ORG_DB_DIR = path.join(__dirname, '../../common/as-org-db');
|
||||
|
||||
// Canonical filename the updater writes to. Manual downloads can use any
|
||||
// *.txt name — findSnapshot picks the newest by mtime regardless.
|
||||
export const AS_ORG_FILE = 'as-org2info.txt';
|
||||
|
||||
const asnToOrgName = new Map();
|
||||
let loadedFrom = null;
|
||||
|
||||
function findSnapshot() {
|
||||
if (!fs.existsSync(AS_ORG_DB_DIR)) return null;
|
||||
const txts = fs.readdirSync(AS_ORG_DB_DIR).filter(f => f.endsWith('.txt'));
|
||||
if (txts.length === 0) return null;
|
||||
const withMtime = txts.map(f => {
|
||||
const full = path.join(AS_ORG_DB_DIR, f);
|
||||
return { full, mtimeMs: fs.statSync(full).mtimeMs };
|
||||
});
|
||||
withMtime.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
||||
return withMtime[0].full;
|
||||
}
|
||||
|
||||
// CAIDA TXT has two sections: org records (5 fields) and ASN records (6
|
||||
// fields). Orgs may appear after ASNs in the stream, so we collect ASN
|
||||
// rows into `pending` first and resolve them after a full scan.
|
||||
function parsePipeText(text) {
|
||||
const orgs = new Map(); // org_id → name
|
||||
const pending = []; // [{asn, org_id}]
|
||||
for (const rawLine of text.split('\n')) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) continue;
|
||||
const parts = line.split('|');
|
||||
if (parts.length === 5) {
|
||||
// org_id | changed | name | country | source
|
||||
if (parts[0] && parts[2]) orgs.set(parts[0], parts[2]);
|
||||
} else if (parts.length === 6) {
|
||||
// aut | changed | aut_name | org_id | opaque_id | source
|
||||
const asn = Number(parts[0]);
|
||||
if (asn && parts[3]) pending.push({ asn, org_id: parts[3] });
|
||||
}
|
||||
}
|
||||
for (const { asn, org_id } of pending) {
|
||||
const name = orgs.get(org_id);
|
||||
if (name) asnToOrgName.set(asn, name);
|
||||
}
|
||||
}
|
||||
|
||||
function loadDatabase() {
|
||||
const filePath = findSnapshot();
|
||||
if (!filePath) {
|
||||
logger.warn({ dir: AS_ORG_DB_DIR }, '⚠️ CAIDA as2org snapshot not found; ASN org-name lookups will fall back to RIPEstat');
|
||||
asnToOrgName.clear();
|
||||
loadedFrom = null;
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
try {
|
||||
const text = fs.readFileSync(filePath, 'utf8');
|
||||
asnToOrgName.clear();
|
||||
parsePipeText(text);
|
||||
// The parser skips what it can't read, so a corrupt file parses to
|
||||
// nothing rather than throwing: that is not a loaded snapshot.
|
||||
if (asnToOrgName.size === 0) {
|
||||
loadedFrom = null;
|
||||
logger.warn({ path: filePath }, '⚠️ CAIDA as2org snapshot holds no ASNs; treating it as missing');
|
||||
return;
|
||||
}
|
||||
loadedFrom = path.basename(filePath);
|
||||
logger.info(`📦 CAIDA as2org loaded (${loadedFrom}) — ${asnToOrgName.size} ASNs in ${Date.now() - start}ms`);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error, path: filePath }, '⚠️ Failed to parse CAIDA as2org snapshot');
|
||||
}
|
||||
}
|
||||
|
||||
loadDatabase();
|
||||
|
||||
/** Reload the snapshot after the dataset updater publishes a fresh file. */
|
||||
export function reloadAsOrgDatabase(reason = 'reload') {
|
||||
logger.info(`🔄 Reloading CAIDA as2org snapshot (${reason})`);
|
||||
loadDatabase();
|
||||
}
|
||||
|
||||
/** Whether a snapshot is loaded (false until the updater has fetched one). */
|
||||
export const isAsOrgLoaded = () => loadedFrom !== null;
|
||||
|
||||
/** Org name for an ASN, or null when the snapshot doesn't have it. */
|
||||
export function lookupAsOrgName(asn) {
|
||||
return asnToOrgName.get(Number(asn)) || null;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Local CAIDA AS Relationships lookup. Snapshot is downloaded
|
||||
// by the dataset updater (server/datasets/datasets.js); this module parses and serves.
|
||||
//
|
||||
// CAIDA as-rel2 row format: `<a>|<b>|<rel>|<source>`
|
||||
// rel = -1 → p2c (a is provider of b) ← providers + customers indexes
|
||||
// rel = 0 → p2p (peering) ← peers index
|
||||
// rel = 1 → s2s (sibling) ← skipped
|
||||
//
|
||||
// Source: https://publicdata.caida.org/datasets/as-relationships/serial-2/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const AS_REL_DB_DIR = path.join(__dirname, '../../common/as-rel-db');
|
||||
export const AS_REL_FILE = 'as-rel2.txt';
|
||||
|
||||
// customer ASN → Set<provider ASN>. Hot path of asn-connectivity.
|
||||
const providersIndex = new Map();
|
||||
|
||||
// ASN → Set<peer ASN>, symmetric p2p. Feeds asn-connectivity's peering edges.
|
||||
const peersIndex = new Map();
|
||||
|
||||
// provider ASN → Set<customer ASN>. Feeds asn-connectivity's neighbour lists.
|
||||
const customersIndex = new Map();
|
||||
|
||||
// ASN → number of distinct customers it provides transit for. Ranking
|
||||
// signal when an intermediate has more providers than MAX_INTERMEDIATE_BRANCH
|
||||
// can show; roughly correlates with customer-cone size.
|
||||
const customerCount = new Map();
|
||||
|
||||
// Tier 1 set, derived from the snapshot rather than hardcoded:
|
||||
// (a) the AS has no providers in CAIDA's p2c topology (nobody sells
|
||||
// transit to it ≈ CAIDA's clique notion), AND
|
||||
// (b) it provides transit to at least TIER1_MIN_CUSTOMERS ASes — filters
|
||||
// out defunct ASNs that look settlement-free only because they're
|
||||
// inactive (e.g. AS1239 Sprint after the T-Mobile merger).
|
||||
// 100 is the empirical sweet spot: captures Telxius/Orange/Liberty
|
||||
// (customer cones 110-340) without admitting clearly defunct ASNs.
|
||||
const TIER1_MIN_CUSTOMERS = 100;
|
||||
const tier1Set = new Set();
|
||||
|
||||
let loadedFrom = null;
|
||||
|
||||
function findSnapshot() {
|
||||
if (!fs.existsSync(AS_REL_DB_DIR)) return null;
|
||||
const txts = fs.readdirSync(AS_REL_DB_DIR).filter(f => f.endsWith('.txt'));
|
||||
if (txts.length === 0) return null;
|
||||
const withMtime = txts.map(f => {
|
||||
const full = path.join(AS_REL_DB_DIR, f);
|
||||
return { full, mtimeMs: fs.statSync(full).mtimeMs };
|
||||
});
|
||||
withMtime.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
||||
return withMtime[0].full;
|
||||
}
|
||||
|
||||
function addToIndex(index, key, value) {
|
||||
let set = index.get(key);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
index.set(key, set);
|
||||
}
|
||||
set.add(value);
|
||||
}
|
||||
|
||||
function parsePipeText(text) {
|
||||
for (const rawLine of text.split('\n')) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) continue;
|
||||
const parts = line.split('|');
|
||||
if (parts.length < 3) continue;
|
||||
const a = Number(parts[0]);
|
||||
const b = Number(parts[1]);
|
||||
if (!a || !b) continue;
|
||||
if (parts[2] === '-1') {
|
||||
addToIndex(providersIndex, b, a);
|
||||
addToIndex(customersIndex, a, b);
|
||||
customerCount.set(a, (customerCount.get(a) || 0) + 1);
|
||||
} else if (parts[2] === '0') {
|
||||
addToIndex(peersIndex, a, b);
|
||||
addToIndex(peersIndex, b, a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function rebuildTier1Set() {
|
||||
tier1Set.clear();
|
||||
for (const [asn, count] of customerCount) {
|
||||
if (providersIndex.has(asn)) continue; // has upstream → not Tier 1
|
||||
if (count >= TIER1_MIN_CUSTOMERS) tier1Set.add(asn);
|
||||
}
|
||||
}
|
||||
|
||||
function loadDatabase() {
|
||||
const filePath = findSnapshot();
|
||||
if (!filePath) {
|
||||
logger.warn({ dir: AS_REL_DB_DIR }, '⚠️ CAIDA as-rel snapshot not found; asn-connectivity will return empty graphs until the updater downloads one');
|
||||
providersIndex.clear();
|
||||
peersIndex.clear();
|
||||
customersIndex.clear();
|
||||
customerCount.clear();
|
||||
tier1Set.clear();
|
||||
loadedFrom = null;
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
try {
|
||||
const text = fs.readFileSync(filePath, 'utf8');
|
||||
providersIndex.clear();
|
||||
peersIndex.clear();
|
||||
customersIndex.clear();
|
||||
customerCount.clear();
|
||||
parsePipeText(text);
|
||||
rebuildTier1Set();
|
||||
// A corrupt file parses to nothing rather than throwing: that is not
|
||||
// a loaded snapshot.
|
||||
if (providersIndex.size === 0 && peersIndex.size === 0) {
|
||||
loadedFrom = null;
|
||||
logger.warn({ path: filePath }, '⚠️ CAIDA as-rel snapshot holds no relationships; treating it as missing');
|
||||
return;
|
||||
}
|
||||
loadedFrom = path.basename(filePath);
|
||||
logger.info(`📦 CAIDA as-rel loaded (${loadedFrom}) — ${providersIndex.size} customers, ${customerCount.size} providers, ${peersIndex.size} peering ASes, ${tier1Set.size} Tier 1s in ${Date.now() - start}ms`);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error, path: filePath }, '⚠️ Failed to parse CAIDA as-rel snapshot');
|
||||
}
|
||||
}
|
||||
|
||||
loadDatabase();
|
||||
|
||||
/** Reload the snapshot after the dataset updater publishes a fresh file. */
|
||||
export function reloadAsRelDatabase(reason = 'reload') {
|
||||
logger.info(`🔄 Reloading CAIDA as-rel snapshot (${reason})`);
|
||||
loadDatabase();
|
||||
}
|
||||
|
||||
/** Providers (transit upstreams) of an ASN. Empty array when not in dataset. */
|
||||
export function providersOf(asn) {
|
||||
const set = providersIndex.get(Number(asn));
|
||||
return set ? [...set] : [];
|
||||
}
|
||||
|
||||
/** Settlement-free peers of an ASN. Empty array when none observed. */
|
||||
export function peersOf(asn) {
|
||||
const set = peersIndex.get(Number(asn));
|
||||
return set ? [...set] : [];
|
||||
}
|
||||
|
||||
/** Customers (transit downstreams) of an ASN. Empty array when it has none. */
|
||||
export const customersOf = (asn) => {
|
||||
const set = customersIndex.get(Number(asn));
|
||||
return set ? [...set] : [];
|
||||
};
|
||||
|
||||
/** How many distinct ASes this one provides transit for. 0 when never a provider. */
|
||||
export function customerCountOf(asn) {
|
||||
return customerCount.get(Number(asn)) || 0;
|
||||
}
|
||||
|
||||
/** Whether a snapshot is loaded (false until the updater has fetched one). */
|
||||
export const isAsRelLoaded = () => loadedFrom !== null;
|
||||
|
||||
/** Whether this ASN is in the CAIDA-derived Tier 1 set. */
|
||||
export function isTier1(asn) {
|
||||
return tier1Set.has(Number(asn));
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
// One engine for every offline snapshot the backend serves — the rows in
|
||||
// server/datasets/datasets.js. A row supplies the domain part: where the newest
|
||||
// remote version is, how to fetch and distill it into the files it
|
||||
// publishes, how to validate them, how to reload them. The engine owns the
|
||||
// rest:
|
||||
// - a cross-process lock per dataset directory (proper-lockfile: the lock
|
||||
// carries a heartbeat, so a crashed holder's lock goes stale on its own);
|
||||
// - a state file per dataset, written atomically (write-file-atomic);
|
||||
// - publishing a row's files together, rolled back if any rename fails;
|
||||
// - the boot download of a snapshot that is missing, unusable or left
|
||||
// half-published, under a timeout;
|
||||
// - the daily schedule (croner), at a fixed wall-clock time, plus a
|
||||
// catch-up run at boot for datasets a downtime kept from their check;
|
||||
// - reloading files published by anyone else — another process, or an
|
||||
// operator dropping files in by hand (watchDatasets).
|
||||
//
|
||||
// Row shape:
|
||||
// id short name, used in logs
|
||||
// dir directory holding the published files, state and lock
|
||||
// files published file names, relative to `dir`
|
||||
// enabled() optional; false skips the row everywhere (a missing key
|
||||
// or credential), as if it weren't registered
|
||||
// findRemote({ signal }) → { identifier, … } — identifier names the remote
|
||||
// version (null when the remote doesn't say: always
|
||||
// fetched); the whole object is handed on to fetch
|
||||
// fetch({ remote, tempDir, signal }) → { [file]: stagedPath } for every
|
||||
// name in `files`
|
||||
// validate(staged) throws to refuse a truncated or corrupt download
|
||||
// reload(reason) swaps the in-memory copy for the published files
|
||||
// isLoaded() optional; whether the reader holds a usable snapshot —
|
||||
// it may accept more than the published names (a
|
||||
// hand-named CAIDA file) or refuse a file it can't parse.
|
||||
// The boot download skips a loaded row. Default: every
|
||||
// published file is present
|
||||
// legacyState optional { file, toState(json) → { identifier, updatedAt } }:
|
||||
// the state file an earlier updater wrote, read once so
|
||||
// an upgrade doesn't re-download
|
||||
// legacyAutoUpdateEnv optional env var name that used to gate this row's
|
||||
// schedule (see isAutoUpdateEnabled)
|
||||
//
|
||||
// State file (`.dataset-state.json`): { identifier, updatedAt, checkedAt } —
|
||||
// the published version, when it was published, when the remote was last
|
||||
// asked about it — plus `publishing: true` while files are in flux.
|
||||
|
||||
import fs from 'fs';
|
||||
import fsp from 'fs/promises';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { setTimeout as sleep } from 'timers/promises';
|
||||
import lockfile from 'proper-lockfile';
|
||||
import writeFileAtomic from 'write-file-atomic';
|
||||
import { Cron } from 'croner';
|
||||
import logger from '../logger.js';
|
||||
import { withCronMonitor } from '../sentry-cron.js';
|
||||
import { createDecompressor } from './decompress.js';
|
||||
import { fetchUpstream } from '../../common/fetch-with-timeout.js';
|
||||
|
||||
export const STATE_FILE = '.dataset-state.json';
|
||||
export const LOCK_FILE = '.dataset.lock';
|
||||
|
||||
// A holder refreshes its lock every minute; one untouched for ten is dead.
|
||||
const LOCK_OPTIONS = { stale: 10 * 60 * 1000, update: 60 * 1000 };
|
||||
const LOCK_RETRY_MS = 2000;
|
||||
const BOOTSTRAP_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
// A scheduled or catch-up update of one dataset, download included; past
|
||||
// this it is aborted so a stalled upstream can't hold up the rest.
|
||||
const UPDATE_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
export const DEFAULT_UPDATE_CRON = '30 4 * * *';
|
||||
const CATCH_UP_DELAY_MS = 60 * 1000;
|
||||
|
||||
// ---------- download helpers for rows ----------
|
||||
|
||||
/**
|
||||
* Stream `url` into `destPath`. fetchUpstream (project User-Agent) times out
|
||||
* the wait for a response; the body — a large archive — is bounded by the
|
||||
* caller's signal instead.
|
||||
*/
|
||||
export const downloadToFile = async (url, destPath, { signal, headers } = {}) => {
|
||||
const response = await fetchUpstream(url, { signal, headers, timeoutMs: 30 * 1000 });
|
||||
if (!response.ok || !response.body) throw new Error(`download failed: HTTP ${response.status}`);
|
||||
await pipeline(Readable.fromWeb(response.body), fs.createWriteStream(destPath), { signal });
|
||||
};
|
||||
|
||||
/** Decompress `srcPath` ('gzip' | 'bzip2') into `destPath`. */
|
||||
export const decompressFile = async (srcPath, destPath, format) => {
|
||||
await pipeline(fs.createReadStream(srcPath), createDecompressor(format), fs.createWriteStream(destPath));
|
||||
};
|
||||
|
||||
// ---------- row state ----------
|
||||
|
||||
export const isRowEnabled = (row) => !row.enabled || Boolean(row.enabled());
|
||||
|
||||
/** Whether every file the row publishes is on disk. */
|
||||
export const isPresent = (row) => row.files.every((file) => fs.existsSync(path.join(row.dir, file)));
|
||||
|
||||
// Whether the boot download can skip the row: its reader has something.
|
||||
const isLoaded = (row) => (row.isLoaded ? row.isLoaded() : isPresent(row));
|
||||
|
||||
/** The row's state, falling back to an earlier updater's state file; {} when neither exists. */
|
||||
export const readState = async (row) => {
|
||||
const read = async (file) => JSON.parse(await fsp.readFile(path.join(row.dir, file), 'utf8'));
|
||||
try {
|
||||
return await read(STATE_FILE);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
}
|
||||
if (!row.legacyState) return {};
|
||||
try {
|
||||
return row.legacyState.toState(await read(row.legacyState.file)) || {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
const writeState = (row, state) =>
|
||||
writeFileAtomic(path.join(row.dir, STATE_FILE), `${JSON.stringify(state, null, 2)}\n`);
|
||||
|
||||
// The published files as this process last loaded them (mtime + size per
|
||||
// file), so the watcher can tell someone else's publish from its own.
|
||||
const loadedFingerprints = new Map();
|
||||
// Rows this process is publishing right now: from the first file move until
|
||||
// the reload has recorded the new fingerprints, the watcher must not take
|
||||
// the changing files for someone else's — it defers its check instead.
|
||||
const ownPublishes = new Set();
|
||||
const fingerprint = (row) => row.files.map((file) => {
|
||||
try {
|
||||
const stat = fs.statSync(path.join(row.dir, file));
|
||||
return `${stat.mtimeMs}:${stat.size}`;
|
||||
} catch {
|
||||
return '-';
|
||||
}
|
||||
}).join('|');
|
||||
// Recorded only once the reload succeeds, so a failed load is retried on
|
||||
// the files' next change.
|
||||
const reloadRow = async (row, reason) => {
|
||||
const current = fingerprint(row);
|
||||
await row.reload(reason);
|
||||
loadedFingerprints.set(row.id, current);
|
||||
};
|
||||
|
||||
// ---------- lock ----------
|
||||
|
||||
// Resolves { release, lost } — `lost` an AbortSignal that fires if the lock
|
||||
// is compromised mid-update (its heartbeat failed; another process may take
|
||||
// it over) — or null when another process holds the lock and `wait` is off.
|
||||
// With `wait`, polls until the lock frees or `signal` aborts (which rejects).
|
||||
const acquireLock = async (row, { wait = false, signal } = {}) => {
|
||||
for (;;) {
|
||||
const lost = new AbortController();
|
||||
try {
|
||||
const release = await lockfile.lock(row.dir, {
|
||||
...LOCK_OPTIONS,
|
||||
lockfilePath: path.join(row.dir, LOCK_FILE),
|
||||
// Never throw (that would kill the server): abort the update
|
||||
// holding the lock instead, before it publishes.
|
||||
onCompromised: (error) => {
|
||||
logger.warn({ err: error, dataset: row.id }, 'dataset lock compromised; aborting the update');
|
||||
lost.abort(error);
|
||||
},
|
||||
});
|
||||
return { release, lost: lost.signal };
|
||||
} catch (error) {
|
||||
if (error.code !== 'ELOCKED') throw error;
|
||||
if (!wait) return null;
|
||||
await sleep(LOCK_RETRY_MS, undefined, { signal });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ---------- publish ----------
|
||||
|
||||
// Copy every staged file next to its target, then rename them all into
|
||||
// place. A failed rename rolls back the ones already done: the previous
|
||||
// file restored from its backup, or a target that didn't exist removed.
|
||||
// `signal` (a timeout, or the lock lost) is checked before every step and
|
||||
// rolls back like a failed rename. Only `lockLost` stops the cleanup too:
|
||||
// another process may be publishing into the same paths, so this one
|
||||
// touches nothing more and leaves the half-done set to the next holder
|
||||
// (updateDataset marked the state in flux first).
|
||||
const publish = async (row, staged, { signal, lockLost }) => {
|
||||
const ownsFiles = () => !lockLost?.aborted;
|
||||
const plan = row.files.map((file) => {
|
||||
const target = path.join(row.dir, file);
|
||||
return { source: staged[file], target, next: `${target}.next`, backup: `${target}.bak`, existed: false, renamed: false };
|
||||
});
|
||||
try {
|
||||
for (const step of plan) {
|
||||
signal?.throwIfAborted();
|
||||
await fsp.copyFile(step.source, step.next);
|
||||
try {
|
||||
await fsp.copyFile(step.target, step.backup);
|
||||
step.existed = true;
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (const step of plan) {
|
||||
signal?.throwIfAborted();
|
||||
await fsp.rename(step.next, step.target);
|
||||
step.renamed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
if (ownsFiles()) {
|
||||
for (const step of plan.filter((s) => s.renamed)) {
|
||||
await (step.existed ? fsp.copyFile(step.backup, step.target) : fsp.rm(step.target, { force: true }))
|
||||
.catch(() => {});
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
if (ownsFiles()) {
|
||||
for (const step of plan) {
|
||||
await fsp.rm(step.next, { force: true });
|
||||
await fsp.rm(step.backup, { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ---------- update ----------
|
||||
|
||||
/**
|
||||
* One locked update of `row`: ask the remote, and when it has a version the
|
||||
* state doesn't (or a file is missing) fetch, validate, publish, record and
|
||||
* reload. Resolves { updated: true, identifier } or { updated: false, reason:
|
||||
* 'not-modified' | 'locked' }; throws when a step fails (nothing published).
|
||||
* `wait` waits out another holder of the lock instead of answering 'locked';
|
||||
* `force` fetches even when the state already records the remote's version.
|
||||
*/
|
||||
export const updateDataset = async (row, { signal, reason = 'auto update', wait = false, force = false } = {}) => {
|
||||
await fsp.mkdir(row.dir, { recursive: true });
|
||||
const lock = await acquireLock(row, { wait, signal });
|
||||
if (!lock) return { updated: false, reason: 'locked' };
|
||||
// Losing the lock stops the download as well as the publish.
|
||||
const work = signal ? AbortSignal.any([signal, lock.lost]) : lock.lost;
|
||||
const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), `myip-dataset-${row.id}-`));
|
||||
try {
|
||||
const state = await readState(row);
|
||||
const remote = await row.findRemote({ signal: work });
|
||||
const now = new Date().toISOString();
|
||||
if (!force && remote.identifier && isPresent(row) && state.identifier === remote.identifier) {
|
||||
work.throwIfAborted();
|
||||
await writeState(row, { ...state, checkedAt: now });
|
||||
return { updated: false, reason: 'not-modified' };
|
||||
}
|
||||
const staged = await row.fetch({ remote, tempDir, signal: work });
|
||||
const missing = row.files.filter((file) => !staged[file]);
|
||||
if (missing.length) throw new Error(`${row.id} fetch produced no ${missing.join(', ')}`);
|
||||
await row.validate(staged);
|
||||
// Files in flux: until the final state below, the record names no
|
||||
// version, so a run that dies or loses the lock mid-publish leaves
|
||||
// nothing the next holder would take for current — it re-fetches and
|
||||
// publishes a whole set, and the next boot does so before trusting
|
||||
// the files.
|
||||
work.throwIfAborted();
|
||||
ownPublishes.add(row.id);
|
||||
await writeState(row, { ...state, identifier: null, publishing: true });
|
||||
await publish(row, staged, { signal: work, lockLost: lock.lost });
|
||||
// A whole set is in place: record it unless the lock went meanwhile.
|
||||
lock.lost.throwIfAborted();
|
||||
await writeState(row, { identifier: remote.identifier, updatedAt: now, checkedAt: now });
|
||||
logger.info({ dataset: row.id, identifier: remote.identifier }, 'dataset updated');
|
||||
await reloadRow(row, reason);
|
||||
return { updated: true, identifier: remote.identifier };
|
||||
} finally {
|
||||
ownPublishes.delete(row.id);
|
||||
await fsp.rm(tempDir, { recursive: true, force: true });
|
||||
await lock.release().catch(() => {});
|
||||
}
|
||||
};
|
||||
|
||||
// Whether a run died or lost its lock mid-publish: the files on disk may
|
||||
// mix two versions. An unreadable state is not taken for one.
|
||||
const wasInterrupted = (row) => readState(row).then((state) => Boolean(state.publishing), () => false);
|
||||
|
||||
/**
|
||||
* Boot: download `row` under a timeout unless its reader already holds a
|
||||
* usable snapshot and no publish was interrupted. Another process (another
|
||||
* backend, a manual run) holding the lock is waited out, and what it
|
||||
* published is loaded. Never throws; resolves { status: 'disabled' |
|
||||
* 'present' | 'downloaded' | 'loaded' | 'no-op' | 'failed' }.
|
||||
*/
|
||||
export const bootstrapDataset = async (row, { timeoutMs = BOOTSTRAP_TIMEOUT_MS } = {}) => {
|
||||
if (!isRowEnabled(row)) return { status: 'disabled' };
|
||||
const interrupted = await wasInterrupted(row);
|
||||
if (isLoaded(row) && !interrupted) return { status: 'present' };
|
||||
// Files already under the published names are unusable or half-published:
|
||||
// a state naming the remote's version is no reason to keep them.
|
||||
const force = isPresent(row);
|
||||
|
||||
const minutes = Math.round(timeoutMs / 60000);
|
||||
const why = interrupted ? 'publish was interrupted' : force ? 'unusable' : 'missing';
|
||||
logger.warn(`📥 Dataset ${row.id} ${why}; downloading (timeout ${minutes} min)...`);
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(new Error('bootstrap timed out')), timeoutMs);
|
||||
timer.unref?.();
|
||||
try {
|
||||
const result = await updateDataset(row, { signal: controller.signal, reason: 'bootstrap', wait: true, force });
|
||||
if (result.updated) {
|
||||
logger.warn(`✅ Dataset ${row.id} downloaded and ready`);
|
||||
return { status: 'downloaded' };
|
||||
}
|
||||
// The remote matched what another process published while we waited.
|
||||
if (isPresent(row)) {
|
||||
await reloadRow(row, 'bootstrap');
|
||||
logger.warn(`✅ Dataset ${row.id} published by another process; loaded`);
|
||||
return { status: 'loaded' };
|
||||
}
|
||||
logger.warn(`⚠️ Dataset ${row.id} bootstrap did not publish (${result.reason})`);
|
||||
return { status: 'no-op' };
|
||||
} catch (error) {
|
||||
const cause = controller.signal.aborted ? `did not complete within ${minutes} min` : error.message;
|
||||
logger.warn({ err: error }, `⚠️ Dataset ${row.id} initial download failed: ${cause}`);
|
||||
return { status: 'failed', error };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
|
||||
/** Bootstrap every row in parallel; never throws. */
|
||||
export const bootstrapDatasets = (rows) => Promise.all(rows.map((row) => bootstrapDataset(row)));
|
||||
|
||||
// ---------- schedule ----------
|
||||
|
||||
/**
|
||||
* Whether the schedule updates `row`. DATASET_AUTO_UPDATE decides when set
|
||||
* ('false' = off); otherwise the row's pre-engine flag when set, so an
|
||||
* existing .env keeps its choice; otherwise on.
|
||||
*/
|
||||
export const isAutoUpdateEnabled = (row, env = process.env) => {
|
||||
const flag = env.DATASET_AUTO_UPDATE;
|
||||
if (flag) return flag !== 'false';
|
||||
const legacy = row.legacyAutoUpdateEnv && env[row.legacyAutoUpdateEnv];
|
||||
if (legacy) return legacy === 'true';
|
||||
return true;
|
||||
};
|
||||
|
||||
// What a scheduled run logs for a row it left alone (a publish logs itself).
|
||||
const SKIP_MESSAGES = {
|
||||
'not-modified': 'dataset up to date; nothing to download',
|
||||
locked: 'dataset update skipped: another process holds the lock',
|
||||
};
|
||||
|
||||
/**
|
||||
* Update each row in turn, each under `timeoutMs`; one failing or stalling
|
||||
* doesn't stop the rest. Every row logs its outcome, so a run that changed
|
||||
* nothing still shows it ran. `timeoutMs` is injectable for tests.
|
||||
*/
|
||||
export const updateDatasets = async (rows, { timeoutMs = UPDATE_TIMEOUT_MS } = {}) => {
|
||||
logger.info({ datasets: rows.map((row) => row.id) }, 'dataset update run started');
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const result = await updateDataset(row, { signal: AbortSignal.timeout(timeoutMs) });
|
||||
if (!result.updated) logger.info({ dataset: row.id }, SKIP_MESSAGES[result.reason]);
|
||||
} catch (error) {
|
||||
logger.error({ err: error, dataset: row.id }, 'dataset update failed');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* The rows of `rows` a downtime kept from their check: a run of `pattern`
|
||||
* fell between the last check (or publish, for a state from before checks
|
||||
* were recorded) and `now` — or there was never one. Exported for tests.
|
||||
*/
|
||||
export const rowsMissingACheck = async (rows, { pattern, now = new Date() }) => {
|
||||
const cron = new Cron(pattern, { paused: true });
|
||||
const stale = [];
|
||||
for (const row of rows) {
|
||||
const { checkedAt, updatedAt } = await readState(row).catch(() => ({}));
|
||||
const last = new Date(checkedAt || updatedAt);
|
||||
const due = Number.isNaN(last.getTime()) ? null : cron.nextRun(last);
|
||||
if (!due || due <= now) stale.push(row);
|
||||
}
|
||||
return stale;
|
||||
};
|
||||
|
||||
/**
|
||||
* Sentry Crons settings for a run updating `rowCount` datasets in turn: its
|
||||
* longest legitimate run is every row at its timeout, plus a margin, so a
|
||||
* healthy run is never marked failed. Exported for tests.
|
||||
*/
|
||||
export const monitorConfigFor = (pattern, rowCount, timezone) => ({
|
||||
schedule: { type: 'crontab', value: pattern },
|
||||
timezone,
|
||||
checkinMargin: 60,
|
||||
maxRuntime: Math.ceil((rowCount * UPDATE_TIMEOUT_MS) / 60000) + 10,
|
||||
});
|
||||
|
||||
let scheduler = null;
|
||||
|
||||
/**
|
||||
* Run the auto-updating rows daily at DATASET_UPDATE_CRON (server-local time,
|
||||
* default 04:30). `protect` skips a tick while the previous run is still
|
||||
* going. Sentry Crons sees the run as one 'dataset-update' check-in.
|
||||
*/
|
||||
export const startDatasetScheduler = (rows) => {
|
||||
if (scheduler) return scheduler;
|
||||
const scheduled = rows.filter((row) => isRowEnabled(row) && isAutoUpdateEnabled(row));
|
||||
if (!scheduled.length) {
|
||||
logger.info('🗓️ Dataset auto update: off');
|
||||
return null;
|
||||
}
|
||||
const pattern = process.env.DATASET_UPDATE_CRON || DEFAULT_UPDATE_CRON;
|
||||
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
||||
scheduler = new Cron(pattern, { protect: true, unref: true }, () => withCronMonitor(
|
||||
'dataset-update',
|
||||
() => updateDatasets(scheduled),
|
||||
monitorConfigFor(pattern, scheduled.length, timezone),
|
||||
).catch((error) => logger.error({ err: error }, 'dataset update run failed')));
|
||||
const next = scheduler.nextRun()?.toLocaleString('en-US', { hour12: false });
|
||||
logger.info(`🗓️ Dataset auto update: ${scheduled.map((row) => row.id).join(', ')} at cron ${pattern} (next ${next})`);
|
||||
|
||||
rowsMissingACheck(scheduled, { pattern }).then((stale) => {
|
||||
if (!stale.length) return;
|
||||
logger.info(`🗓️ Dataset catch-up in ${CATCH_UP_DELAY_MS / 1000}s: ${stale.map((row) => row.id).join(', ')}`);
|
||||
setTimeout(() => updateDatasets(stale), CATCH_UP_DELAY_MS).unref?.();
|
||||
});
|
||||
return scheduler;
|
||||
};
|
||||
|
||||
// ---------- watch ----------
|
||||
|
||||
/**
|
||||
* Reload a row when its published files change under it — another process
|
||||
* published, or someone dropped files in by hand — but not after this
|
||||
* process's own publish (already reloaded). Polls every `intervalMs`; the
|
||||
* files must then hold still for `settleMs` (a copy still being written
|
||||
* keeps moving) and pass the row's validator before the reload, so a
|
||||
* partial or corrupt file never replaces healthy in-memory data. Watches
|
||||
* every row, enabled or not: hand-placed files need no credentials. Returns
|
||||
* a stop function. Options are injectable for tests.
|
||||
*/
|
||||
export const watchDatasets = (rows, { intervalMs = 5000, settleMs = 1000 } = {}) => {
|
||||
const stops = [];
|
||||
for (const row of rows) {
|
||||
if (!loadedFingerprints.has(row.id)) loadedFingerprints.set(row.id, fingerprint(row));
|
||||
let timer = null;
|
||||
let seen = null;
|
||||
const settle = async () => {
|
||||
const current = fingerprint(row);
|
||||
if (current !== seen) {
|
||||
seen = current;
|
||||
timer = setTimeout(settle, settleMs);
|
||||
timer.unref?.();
|
||||
return;
|
||||
}
|
||||
// This process's own publish records its files once reloaded;
|
||||
// look again after it, so a change made meanwhile still counts.
|
||||
if (ownPublishes.has(row.id)) {
|
||||
timer = setTimeout(settle, settleMs);
|
||||
timer.unref?.();
|
||||
return;
|
||||
}
|
||||
if (current === loadedFingerprints.get(row.id) || !isPresent(row)) return;
|
||||
const published = Object.fromEntries(row.files.map((file) => [file, path.join(row.dir, file)]));
|
||||
try {
|
||||
await row.validate(published);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error, dataset: row.id }, 'dataset files changed on disk but failed validation; not reloading');
|
||||
return;
|
||||
}
|
||||
logger.info({ dataset: row.id }, 'dataset files changed on disk; reloading');
|
||||
await reloadRow(row, 'file change').catch((error) => {
|
||||
logger.warn({ err: error, dataset: row.id }, 'dataset reload after a file change failed');
|
||||
});
|
||||
};
|
||||
const check = () => {
|
||||
clearTimeout(timer);
|
||||
seen = fingerprint(row);
|
||||
timer = setTimeout(settle, settleMs);
|
||||
timer.unref?.();
|
||||
};
|
||||
for (const file of row.files) {
|
||||
const target = path.join(row.dir, file);
|
||||
fs.watchFile(target, { interval: intervalMs, persistent: false }, check);
|
||||
stops.push(() => fs.unwatchFile(target, check));
|
||||
}
|
||||
stops.push(() => clearTimeout(timer));
|
||||
}
|
||||
return () => stops.forEach((stop) => stop());
|
||||
};
|
||||
@@ -0,0 +1,307 @@
|
||||
// The offline datasets the backend serves, as rows for the engine in
|
||||
// server/datasets/dataset-updater.js (row shape documented there). Each row holds
|
||||
// only the dataset's own logic: where its newest remote version is, how to
|
||||
// turn the download into the published files, a validator that refuses a
|
||||
// truncated or corrupt download, and the reload of its in-memory copy.
|
||||
//
|
||||
// maxmind GeoLite2 City + ASN, published server/datasets/maxmind-service.js
|
||||
// together; only with MaxMind credentials
|
||||
// as2org CAIDA AS → org name server/datasets/as-org-db.js
|
||||
// as-rel CAIDA AS relationships server/datasets/as-rel-db.js
|
||||
// peeringdb PeeringDB dump (CAIDA mirror), server/datasets/peeringdb-db.js
|
||||
// distilled to a per-ASN index; only with the Cloudflare key,
|
||||
// as the ASN Profile is hidden without it
|
||||
// oui IEEE MAC registries (MA-L / MA-M server/datasets/oui-db.js
|
||||
// / MA-S / IAB / CID), published together as downloaded
|
||||
//
|
||||
// Sources: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data ·
|
||||
// https://publicdata.caida.org/datasets/ (CAIDA AUA) · https://www.peeringdb.com ·
|
||||
// https://standards.ieee.org/products-programs/regauth/
|
||||
|
||||
import fsp from 'fs/promises';
|
||||
import path from 'path';
|
||||
import * as tar from 'tar';
|
||||
import maxmind from 'maxmind';
|
||||
import logger from '../logger.js';
|
||||
import { fetchUpstream } from '../../common/fetch-with-timeout.js';
|
||||
import { downloadToFile, decompressFile } from './dataset-updater.js';
|
||||
import { AS_ORG_DB_DIR, AS_ORG_FILE, reloadAsOrgDatabase, isAsOrgLoaded } from './as-org-db.js';
|
||||
import { AS_REL_DB_DIR, AS_REL_FILE, reloadAsRelDatabase, isAsRelLoaded } from './as-rel-db.js';
|
||||
import { PEERINGDB_DB_DIR, PEERINGDB_FILE, readPeeringdbIndex, reloadPeeringdbDatabase, isPeeringdbLoaded } from './peeringdb-db.js';
|
||||
import { distillPeeringdbDump } from './peeringdb-distill.js';
|
||||
import { hasRadarApiKey } from '../cf-radar.js';
|
||||
import { MAXMIND_DB_DIR, MAXMIND_CITY_DB, MAXMIND_ASN_DB, reloadMaxMindDatabases, isMaxMindReady } from './maxmind-service.js';
|
||||
import { OUI_DB_DIR, OUI_REGISTRIES, OUI_FILES, readOuiRegistries, reloadOuiDatabase, isOuiLoaded } from './oui-db.js';
|
||||
|
||||
// The pre-engine CAIDA updater's state: same { identifier, updatedAt } shape.
|
||||
const CAIDA_LEGACY_STATE = { file: '.caida-update-state.json', toState: (json) => json };
|
||||
|
||||
// Download an archive and decompress it to the one file a row publishes.
|
||||
const fetchCompressed = (file, format) => async ({ remote, tempDir, signal }) => {
|
||||
const archive = path.join(tempDir, 'archive');
|
||||
const staged = path.join(tempDir, file);
|
||||
await downloadToFile(remote.url, archive, { signal });
|
||||
await decompressFile(archive, staged, format);
|
||||
return { [file]: staged };
|
||||
};
|
||||
|
||||
// Last-Modified of each file a row publishes together, joined: any one
|
||||
// changing is a new version. null when a file doesn't say — an unknown
|
||||
// version, always fetched rather than taken for unchanged. Exported for
|
||||
// tests (and the legacy MaxMind state mapping below).
|
||||
export const joinedIdentifier = (lastModifieds) =>
|
||||
(lastModifieds.every(Boolean) ? lastModifieds.join(' | ') : null);
|
||||
|
||||
// ---------- validators ----------
|
||||
|
||||
// Healthy as2org resolves ~110k ASNs; <50k indicates truncation or schema change.
|
||||
const validateAsOrg = async ({ [AS_ORG_FILE]: stagedPath }) => {
|
||||
const MIN_VALID = 50000;
|
||||
const text = await fsp.readFile(stagedPath, 'utf8');
|
||||
const orgs = new Map();
|
||||
const pending = [];
|
||||
for (const rawLine of text.split('\n')) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) continue;
|
||||
const parts = line.split('|');
|
||||
if (parts.length === 5 && parts[0] && parts[2]) {
|
||||
orgs.set(parts[0], parts[2]);
|
||||
} else if (parts.length === 6 && Number(parts[0]) && parts[3]) {
|
||||
pending.push(parts[3]);
|
||||
}
|
||||
}
|
||||
let resolved = 0;
|
||||
for (const orgId of pending) if (orgs.has(orgId)) resolved++;
|
||||
if (resolved < MIN_VALID) {
|
||||
throw new Error(`Staged as2org has only ${resolved} resolvable ASNs; refusing to publish`);
|
||||
}
|
||||
};
|
||||
|
||||
// Healthy as-rel2 has ~400k p2c rows; <100k indicates truncation.
|
||||
const validateAsRel = async ({ [AS_REL_FILE]: stagedPath }) => {
|
||||
const MIN_VALID = 100000;
|
||||
const text = await fsp.readFile(stagedPath, 'utf8');
|
||||
let p2cRows = 0;
|
||||
for (const rawLine of text.split('\n')) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) continue;
|
||||
const parts = line.split('|');
|
||||
if (parts.length >= 3 && parts[2] === '-1') p2cRows++;
|
||||
}
|
||||
if (p2cRows < MIN_VALID) {
|
||||
throw new Error(`Staged as-rel has only ${p2cRows} p2c rows; refusing to publish`);
|
||||
}
|
||||
};
|
||||
|
||||
// Healthy index has ~33k networks; <20k indicates truncation or schema change.
|
||||
const validatePeeringdb = async ({ [PEERINGDB_FILE]: stagedPath }) => {
|
||||
const MIN_VALID = 20000;
|
||||
const { nets } = readPeeringdbIndex(stagedPath);
|
||||
if (nets.size < MIN_VALID) {
|
||||
throw new Error(`Staged PeeringDB index has only ${nets.size} networks; refusing to publish`);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------- remote lookups ----------
|
||||
|
||||
// CAIDA mirrors the full PeeringDB dump daily (a day late) under YYYY/MM/.
|
||||
// List the current UTC month, then the previous one for the days before the
|
||||
// month's first dump lands; the newest filename wins. Exported for tests.
|
||||
export const findPeeringdbDump = async ({ signal, now = new Date() } = {}) => {
|
||||
const base = 'https://publicdata.caida.org/datasets/peeringdb/';
|
||||
for (const back of [0, 1]) {
|
||||
const month = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - back, 1));
|
||||
const dir = `${base}${month.getUTCFullYear()}/${String(month.getUTCMonth() + 1).padStart(2, '0')}/`;
|
||||
const res = await fetchUpstream(dir, { signal });
|
||||
if (res.status === 404) continue;
|
||||
if (!res.ok) throw new Error(`Directory listing failed: HTTP ${res.status}`);
|
||||
const html = await res.text();
|
||||
const names = [...html.matchAll(/\bpeeringdb_2_dump_\d{4}_\d{2}_\d{2}\.json\b/g)].map((m) => m[0]);
|
||||
if (names.length === 0) continue;
|
||||
names.sort((a, b) => b.localeCompare(a));
|
||||
return { url: dir + names[0], identifier: names[0] };
|
||||
}
|
||||
throw new Error('No PeeringDB dump in the current or previous month listing');
|
||||
};
|
||||
|
||||
// ---------- MaxMind ----------
|
||||
|
||||
// The two editions, published together: one version identifier covers both.
|
||||
const MAXMIND_EDITIONS = [
|
||||
{ editionId: 'GeoLite2-City', file: MAXMIND_CITY_DB },
|
||||
{ editionId: 'GeoLite2-ASN', file: MAXMIND_ASN_DB },
|
||||
];
|
||||
|
||||
export const hasMaxMindCredentials = () =>
|
||||
Boolean(process.env.MAXMIND_ACCOUNT_ID && process.env.MAXMIND_LICENSE_KEY);
|
||||
|
||||
const maxmindUrl = (editionId) => `https://download.maxmind.com/geoip/databases/${editionId}/download?suffix=tar.gz`;
|
||||
const maxmindAuth = () => ({
|
||||
Authorization: `Basic ${Buffer.from(`${process.env.MAXMIND_ACCOUNT_ID}:${process.env.MAXMIND_LICENSE_KEY}`).toString('base64')}`,
|
||||
});
|
||||
|
||||
// The pre-engine MaxMind updater kept { [editionId]: { lastModified, updatedAt } }.
|
||||
const MAXMIND_LEGACY_STATE = {
|
||||
file: '.maxmind-update-state.json',
|
||||
toState: (json) => ({
|
||||
identifier: joinedIdentifier(MAXMIND_EDITIONS.map(({ editionId }) => json[editionId]?.lastModified)),
|
||||
updatedAt: json['GeoLite2-City']?.updatedAt,
|
||||
}),
|
||||
};
|
||||
|
||||
// The archive nests the .mmdb in a dated directory; find it by name.
|
||||
const findFile = async (dir, name) => {
|
||||
for (const entry of await fsp.readdir(dir, { withFileTypes: true })) {
|
||||
const entryPath = path.join(dir, entry.name);
|
||||
if (entry.isFile() && entry.name === name) return entryPath;
|
||||
if (entry.isDirectory()) {
|
||||
const found = await findFile(entryPath, name);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// ---------- rows ----------
|
||||
|
||||
// Exported for tests (rows are re-pointed at a temp dir there).
|
||||
export const datasets = [
|
||||
{
|
||||
id: 'maxmind',
|
||||
dir: MAXMIND_DB_DIR,
|
||||
files: MAXMIND_EDITIONS.map(({ file }) => file),
|
||||
enabled: hasMaxMindCredentials,
|
||||
legacyState: MAXMIND_LEGACY_STATE,
|
||||
legacyAutoUpdateEnv: 'MAXMIND_AUTO_UPDATE',
|
||||
findRemote: async ({ signal } = {}) => {
|
||||
const lastModifieds = [];
|
||||
for (const { editionId } of MAXMIND_EDITIONS) {
|
||||
const res = await fetchUpstream(maxmindUrl(editionId), { method: 'HEAD', headers: maxmindAuth(), signal });
|
||||
if (!res.ok) throw new Error(`Failed to check ${editionId}: HTTP ${res.status}`);
|
||||
lastModifieds.push(res.headers.get('last-modified'));
|
||||
}
|
||||
return { identifier: joinedIdentifier(lastModifieds) };
|
||||
},
|
||||
fetch: async ({ tempDir, signal }) => {
|
||||
const staged = {};
|
||||
for (const { editionId, file } of MAXMIND_EDITIONS) {
|
||||
const archive = path.join(tempDir, `${editionId}.tar.gz`);
|
||||
const extractDir = path.join(tempDir, editionId);
|
||||
await fsp.mkdir(extractDir);
|
||||
await downloadToFile(maxmindUrl(editionId), archive, { signal, headers: maxmindAuth() });
|
||||
await tar.x({ file: archive, cwd: extractDir });
|
||||
staged[file] = await findFile(extractDir, file);
|
||||
if (!staged[file]) throw new Error(`${editionId} archive did not contain ${file}`);
|
||||
}
|
||||
return staged;
|
||||
},
|
||||
// Both must open as MaxMind databases.
|
||||
validate: async (staged) => {
|
||||
await Promise.all(MAXMIND_EDITIONS.map(({ file }) => maxmind.open(staged[file])));
|
||||
},
|
||||
reload: reloadMaxMindDatabases,
|
||||
isLoaded: isMaxMindReady,
|
||||
},
|
||||
{
|
||||
id: 'as2org',
|
||||
dir: AS_ORG_DB_DIR,
|
||||
files: [AS_ORG_FILE],
|
||||
// The reader takes the newest *.txt, whatever its name.
|
||||
isLoaded: isAsOrgLoaded,
|
||||
legacyState: CAIDA_LEGACY_STATE,
|
||||
legacyAutoUpdateEnv: 'CAIDA_AUTO_UPDATE',
|
||||
// Stable 'latest' symlink server-side → HEAD + Last-Modified is enough
|
||||
// (no header → null identifier → always fetched).
|
||||
findRemote: async ({ signal } = {}) => {
|
||||
const url = 'https://publicdata.caida.org/datasets/as-organizations/latest.as-org2info.txt.gz';
|
||||
const res = await fetchUpstream(url, { method: 'HEAD', signal });
|
||||
if (!res.ok) throw new Error(`HEAD failed: HTTP ${res.status}`);
|
||||
return { url, identifier: res.headers.get('last-modified') };
|
||||
},
|
||||
fetch: fetchCompressed(AS_ORG_FILE, 'gzip'),
|
||||
validate: validateAsOrg,
|
||||
reload: reloadAsOrgDatabase,
|
||||
},
|
||||
{
|
||||
id: 'as-rel',
|
||||
dir: AS_REL_DB_DIR,
|
||||
files: [AS_REL_FILE],
|
||||
// The reader takes the newest *.txt, whatever its name.
|
||||
isLoaded: isAsRelLoaded,
|
||||
legacyState: CAIDA_LEGACY_STATE,
|
||||
legacyAutoUpdateEnv: 'CAIDA_AUTO_UPDATE',
|
||||
// No 'latest' symlink — scrape the directory listing; YYYYMMDD
|
||||
// filenames sort chronologically, the newest wins.
|
||||
findRemote: async ({ signal } = {}) => {
|
||||
const dir = 'https://publicdata.caida.org/datasets/as-relationships/serial-2/';
|
||||
const res = await fetchUpstream(dir, { signal });
|
||||
if (!res.ok) throw new Error(`Directory listing failed: HTTP ${res.status}`);
|
||||
const html = await res.text();
|
||||
const matches = [...html.matchAll(/\b(\d{8})\.as-rel2\.txt\.bz2\b/g)];
|
||||
if (matches.length === 0) throw new Error('No as-rel2.txt.bz2 found in listing');
|
||||
matches.sort((a, b) => b[1].localeCompare(a[1]));
|
||||
const filename = matches[0][0];
|
||||
return { url: dir + filename, identifier: filename };
|
||||
},
|
||||
fetch: fetchCompressed(AS_REL_FILE, 'bzip2'),
|
||||
validate: validateAsRel,
|
||||
reload: reloadAsRelDatabase,
|
||||
},
|
||||
{
|
||||
id: 'peeringdb',
|
||||
dir: PEERINGDB_DB_DIR,
|
||||
files: [PEERINGDB_FILE],
|
||||
enabled: hasRadarApiKey,
|
||||
legacyState: CAIDA_LEGACY_STATE,
|
||||
legacyAutoUpdateEnv: 'CAIDA_AUTO_UPDATE',
|
||||
findRemote: findPeeringdbDump,
|
||||
// ~116 MB raw dump → ~6 MB index; only the index is published.
|
||||
fetch: async ({ remote, tempDir, signal }) => {
|
||||
const raw = path.join(tempDir, 'dump.json');
|
||||
const staged = path.join(tempDir, PEERINGDB_FILE);
|
||||
await downloadToFile(remote.url, raw, { signal });
|
||||
try {
|
||||
const counts = await distillPeeringdbDump(raw, staged, { signal });
|
||||
logger.info({ dataset: 'peeringdb', ...counts }, 'PeeringDB dump distilled');
|
||||
} catch (error) {
|
||||
throw new Error(`peeringdb distill failed: ${error.message}`);
|
||||
} finally {
|
||||
await fsp.rm(raw, { force: true });
|
||||
}
|
||||
return { [PEERINGDB_FILE]: staged };
|
||||
},
|
||||
validate: validatePeeringdb,
|
||||
reload: reloadPeeringdbDatabase,
|
||||
isLoaded: isPeeringdbLoaded,
|
||||
},
|
||||
{
|
||||
id: 'oui',
|
||||
dir: OUI_DB_DIR,
|
||||
files: OUI_FILES,
|
||||
// IEEE regenerates the files daily; a file's Last-Modified is its version.
|
||||
findRemote: async ({ signal } = {}) => {
|
||||
const lastModifieds = [];
|
||||
for (const { file, url } of OUI_REGISTRIES) {
|
||||
const res = await fetchUpstream(url, { method: 'HEAD', signal });
|
||||
if (!res.ok) throw new Error(`Failed to check ${file}: HTTP ${res.status}`);
|
||||
lastModifieds.push(res.headers.get('last-modified'));
|
||||
}
|
||||
return { identifier: joinedIdentifier(lastModifieds) };
|
||||
},
|
||||
// Plain GETs: the IEEE server rejects ranged requests.
|
||||
fetch: async ({ tempDir, signal }) => {
|
||||
const staged = {};
|
||||
for (const { file, url } of OUI_REGISTRIES) {
|
||||
staged[file] = path.join(tempDir, file);
|
||||
await downloadToFile(url, staged[file], { signal });
|
||||
}
|
||||
return staged;
|
||||
},
|
||||
// Every file must parse as its registry, above its row floor.
|
||||
validate: async (staged) => {
|
||||
readOuiRegistries((file) => staged[file]);
|
||||
},
|
||||
reload: reloadOuiDatabase,
|
||||
isLoaded: isOuiLoaded,
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Single entry point for stream decompression. New formats become a switch
|
||||
// case here rather than a new dependency in every updater file.
|
||||
//
|
||||
// gzip → Node built-in zlib (no dep)
|
||||
// bzip2 → unbzip2-stream (Node has no native bzip2)
|
||||
|
||||
import zlib from 'zlib';
|
||||
import unbzip2Stream from 'unbzip2-stream';
|
||||
|
||||
export function createDecompressor(format) {
|
||||
switch (format) {
|
||||
case 'gzip': return zlib.createGunzip();
|
||||
case 'bzip2': return unbzip2Stream();
|
||||
default:
|
||||
throw new Error(`Unsupported decompression format: ${format}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import maxmind from 'maxmind';
|
||||
import logger from '../logger.js';
|
||||
import { matchLocale } from '../../common/locale-registry.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
export const MAXMIND_DB_DIR = path.resolve(__dirname, '../../common/maxmind-db');
|
||||
export const MAXMIND_CITY_DB = 'GeoLite2-City.mmdb';
|
||||
export const MAXMIND_ASN_DB = 'GeoLite2-ASN.mmdb';
|
||||
|
||||
const cityDbPath = path.join(MAXMIND_DB_DIR, MAXMIND_CITY_DB);
|
||||
const asnDbPath = path.join(MAXMIND_DB_DIR, MAXMIND_ASN_DB);
|
||||
|
||||
// The languages the City database carries `names` for — a property of the data
|
||||
// itself, not of the UI: registering a new front-end locale changes nothing
|
||||
// here, and this list only moves if MaxMind's own coverage does.
|
||||
export const SUPPORTED_LANGS = ['de', 'en', 'es', 'fr', 'ja', 'pt-BR', 'ru', 'zh-CN'];
|
||||
|
||||
// Resolve any BCP-47 tag onto that set: exact match (canonical spelling wins),
|
||||
// then the base language, then a sibling of the same family — `zh-TW` reads
|
||||
// `zh-CN` names rather than falling all the way to English. Anything else, junk
|
||||
// included, lands on `en`. Family resolution is defined once, in the locale
|
||||
// registry; only the resolver is borrowed, never its list of UI locales.
|
||||
export const normalizeLang = (tag) => {
|
||||
if (typeof tag !== 'string') return 'en';
|
||||
return matchLocale(tag.trim(), SUPPORTED_LANGS) ?? 'en';
|
||||
};
|
||||
|
||||
let cityLookup = null;
|
||||
let asnLookup = null;
|
||||
let reloadPromise = null;
|
||||
|
||||
/**
|
||||
* Return the canonical MaxMind database paths.
|
||||
*/
|
||||
export function getMaxMindDbPaths() {
|
||||
return {
|
||||
dbDir: MAXMIND_DB_DIR,
|
||||
cityDbPath,
|
||||
asnDbPath,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Report whether both City and ASN readers are ready to serve lookups.
|
||||
*/
|
||||
export function isMaxMindReady() {
|
||||
return Boolean(cityLookup && asnLookup);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open fresh MaxMind readers from the provided database paths.
|
||||
*/
|
||||
export async function openMaxMindReaders(dbPaths = getMaxMindDbPaths()) {
|
||||
const [city, asn] = await Promise.all([
|
||||
maxmind.open(dbPaths.cityDbPath),
|
||||
maxmind.open(dbPaths.asnDbPath),
|
||||
]);
|
||||
|
||||
return { city, asn };
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the active MaxMind readers after both databases are opened successfully.
|
||||
*/
|
||||
export async function reloadMaxMindDatabases(reason = 'manual') {
|
||||
if (reloadPromise) {
|
||||
return reloadPromise;
|
||||
}
|
||||
|
||||
reloadPromise = openMaxMindReaders()
|
||||
.then(({ city, asn }) => {
|
||||
cityLookup = city;
|
||||
asnLookup = asn;
|
||||
logger.info(`📦 MaxMind databases loaded (${reason})`);
|
||||
return true;
|
||||
})
|
||||
.catch(error => {
|
||||
logger.error({ err: error, reason }, 'Failed to load MaxMind databases');
|
||||
throw error;
|
||||
})
|
||||
.finally(() => {
|
||||
reloadPromise = null;
|
||||
});
|
||||
|
||||
return reloadPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bare ISO country code of an IP (null when unknown) plus the prefix length
|
||||
* of the MaxMind network holding it, so a caller can walk a range one network
|
||||
* at a time — the bgp-prefixes Radar view does, thousands of times per
|
||||
* request. No formatting, no 503 throw: null when there is no answer — the
|
||||
* City database isn't loaded, or the lookup failed — so the caller can tell
|
||||
* it from an address MaxMind places in no country ({ country: null }).
|
||||
*/
|
||||
export const lookupCountryRange = (ip) => {
|
||||
if (!cityLookup) return null;
|
||||
try {
|
||||
const [record, prefixLength] = cityLookup.getWithPrefixLength(ip);
|
||||
return { country: record?.country?.iso_code || null, prefixLength };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Look up an IP address and return the API response shape expected by the frontend.
|
||||
* `lang` is any raw tag — normalization happens here, so no caller can skip it.
|
||||
*/
|
||||
export function lookupMaxMind(ip, lang) {
|
||||
if (!isMaxMindReady()) {
|
||||
const error = new Error('MaxMind database is not ready');
|
||||
error.statusCode = 503;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const city = cityLookup.get(ip);
|
||||
const asn = asnLookup.get(ip);
|
||||
|
||||
return formatMaxMindResult(ip, normalizeLang(lang), city, asn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the requested localized name with English and N/A fallbacks.
|
||||
*/
|
||||
function getLocalizedName(names, lang) {
|
||||
if (!names) {
|
||||
return 'N/A';
|
||||
}
|
||||
|
||||
return names[lang] || names.en || 'N/A';
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize MaxMind City and ASN records into this API's stable JSON format.
|
||||
*/
|
||||
function formatMaxMindResult(ip, lang, city, asn) {
|
||||
city = city || {};
|
||||
asn = asn || {};
|
||||
|
||||
return {
|
||||
ip,
|
||||
city: city.city ? getLocalizedName(city.city.names, lang) : 'N/A',
|
||||
region: city.subdivisions?.[0] ? getLocalizedName(city.subdivisions[0].names, lang) : 'N/A',
|
||||
country: city.country ? city.country.iso_code : 'N/A',
|
||||
country_name: city.country ? getLocalizedName(city.country.names, lang) : 'N/A',
|
||||
country_code: city.country ? city.country.iso_code : 'N/A',
|
||||
latitude: city.location ? city.location.latitude : 'N/A',
|
||||
longitude: city.location ? city.location.longitude : 'N/A',
|
||||
asn: asn.autonomous_system_number ? `AS${asn.autonomous_system_number}` : 'N/A',
|
||||
org: asn.autonomous_system_organization ? asn.autonomous_system_organization : 'N/A',
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
// Local IEEE MAC address block lookup for /api/macchecker. server/datasets/datasets.js
|
||||
// downloads the five public IEEE registries as published (CSV, one per
|
||||
// block size); this module parses them into one map keyed by assignment
|
||||
// hex and answers the longest assignment covering a query.
|
||||
//
|
||||
// MA-L oui.csv 24-bit vendor prefix (the classic OUI)
|
||||
// MA-M mam.csv 28-bit block
|
||||
// MA-S oui36.csv 36-bit block
|
||||
// IAB iab.csv 36-bit block (retired; existing assignments remain)
|
||||
// CID cid.csv 24-bit Company ID (locally administered space)
|
||||
//
|
||||
// MA-M / MA-S / IAB blocks sit inside MA-L prefixes registered to "IEEE
|
||||
// Registration Authority", so the longest assignment wins. Each CSV row is
|
||||
// `Registry,Assignment,Organization Name,Organization Address`; the country
|
||||
// is not a column, it is read off the end of the address.
|
||||
//
|
||||
// Source: https://standards.ieee.org/products-programs/regauth/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import logger from '../logger.js';
|
||||
import { MAC_HEX_LENGTH } from '../../common/mac-input.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const OUI_DB_DIR = path.join(__dirname, '../../common/oui-db');
|
||||
|
||||
const IEEE_BASE = 'https://standards-oui.ieee.org';
|
||||
|
||||
// `min` is the row floor below which a download counts as truncated
|
||||
// (roughly 70% of the 2026 sizes; IAB is frozen at ~4.6k).
|
||||
export const OUI_REGISTRIES = [
|
||||
{ registry: 'MA-L', file: 'oui.csv', url: `${IEEE_BASE}/oui/oui.csv`, hexLength: 6, min: 28000 },
|
||||
{ registry: 'MA-M', file: 'mam.csv', url: `${IEEE_BASE}/oui28/mam.csv`, hexLength: 7, min: 4500 },
|
||||
{ registry: 'MA-S', file: 'oui36.csv', url: `${IEEE_BASE}/oui36/oui36.csv`, hexLength: 9, min: 5000 },
|
||||
{ registry: 'IAB', file: 'iab.csv', url: `${IEEE_BASE}/iab/iab.csv`, hexLength: 9, min: 3200 },
|
||||
{ registry: 'CID', file: 'cid.csv', url: `${IEEE_BASE}/cid/cid.csv`, hexLength: 6, min: 150 },
|
||||
];
|
||||
export const OUI_FILES = OUI_REGISTRIES.map(({ file }) => file);
|
||||
|
||||
// Longest first: a sub-block beats the IEEE RA prefix holding it.
|
||||
const HEX_LENGTHS = [...new Set(OUI_REGISTRIES.map(({ hexLength }) => hexLength))].sort((a, b) => b - a);
|
||||
|
||||
const CSV_HEADER = 'Registry,Assignment,Organization Name,Organization Address';
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Parse */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* RFC 4180 CSV text → rows of fields (quoted fields may hold commas, `""`,
|
||||
* newlines). Throws on text that ends inside a quoted field.
|
||||
*/
|
||||
export const parseCsv = (text) => {
|
||||
const rows = [];
|
||||
let row = [];
|
||||
let field = '';
|
||||
let quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (quoted) {
|
||||
if (c !== '"') field += c;
|
||||
else if (text[i + 1] === '"') { field += '"'; i++; }
|
||||
else quoted = false;
|
||||
} else if (c === '"') quoted = true;
|
||||
else if (c === ',') { row.push(field); field = ''; }
|
||||
else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
|
||||
else if (c !== '\r') field += c;
|
||||
}
|
||||
if (quoted) throw new Error('CSV ends inside a quoted field');
|
||||
if (field || row.length) { row.push(field); rows.push(row); }
|
||||
return rows;
|
||||
};
|
||||
|
||||
const regionNames = new Intl.DisplayNames(['en'], { type: 'region', fallback: 'none' });
|
||||
const isCountryCode = (token) => /^[A-Z]{2}$/.test(token || '') && Boolean(regionNames.of(token));
|
||||
const hasDigit = (token) => /\d/.test(token || '');
|
||||
|
||||
// Countries whose postal code is followed by letters that can read as a
|
||||
// country code: Dutch postcodes ("5633 AD"), Italian CAP + province ("10077 TO").
|
||||
const LETTERED_POSTCODE = new Set(['NL', 'IT']);
|
||||
|
||||
/**
|
||||
* ISO country code at the end of an IEEE address, or null. The address ends
|
||||
* `… <country> <postal code>`, where the postal code is zero to two tokens
|
||||
* of free text ("94568", "5633 AD", "R.O.C."). A code followed by a token
|
||||
* with a digit is the country — unless another code trails the postal code
|
||||
* ("Dover DE 19901 US", a state before the country), which then wins, save
|
||||
* after a lettered-postcode country. Otherwise the rightmost code.
|
||||
*/
|
||||
export const countryFromAddress = (address) => {
|
||||
const tokens = String(address || '').trim().split(/\s+/).slice(-3);
|
||||
for (let i = 0; i < tokens.length - 1; i++) {
|
||||
if (!isCountryCode(tokens[i]) || !hasDigit(tokens[i + 1])) continue;
|
||||
const trailing = tokens.slice(i + 2).find(isCountryCode);
|
||||
return trailing && !LETTERED_POSTCODE.has(tokens[i]) ? trailing : tokens[i];
|
||||
}
|
||||
for (let i = tokens.length - 1; i >= 0; i--) {
|
||||
if (isCountryCode(tokens[i])) return tokens[i];
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* One registry's CSV text → [assignment, { registry, company, address,
|
||||
* country }] entries. Throws on a file that isn't an IEEE registry CSV, or
|
||||
* one cut short.
|
||||
*/
|
||||
export const parseRegistry = (text, { registry, hexLength }) => {
|
||||
const rows = parseCsv(text.replace(/^\uFEFF/, ''));
|
||||
if (rows[0]?.join(',').trim() !== CSV_HEADER) throw new Error(`${registry}: unexpected CSV header`);
|
||||
// IEEE ends every row, the last included, with CRLF: a file that stops
|
||||
// mid-row was cut short.
|
||||
if (!text.endsWith('\n')) throw new Error(`${registry}: file ends mid-row (truncated)`);
|
||||
const entries = [];
|
||||
for (const [rowRegistry, assignment, company, address] of rows.slice(1)) {
|
||||
const hex = (assignment || '').trim().toUpperCase();
|
||||
if (rowRegistry?.trim() !== registry || hex.length !== hexLength || !/^[0-9A-F]+$/.test(hex)) continue;
|
||||
const cleanAddress = (address || '').trim().replace(/\s+/g, ' ');
|
||||
entries.push([hex, {
|
||||
registry,
|
||||
company: (company || '').trim(),
|
||||
address: cleanAddress,
|
||||
country: countryFromAddress(cleanAddress),
|
||||
}]);
|
||||
}
|
||||
return entries;
|
||||
};
|
||||
|
||||
/**
|
||||
* Read every registry file into one Map; `pathOf(file)` locates each.
|
||||
* Throws when a file is missing, malformed or below its row floor.
|
||||
*/
|
||||
export const readOuiRegistries = (pathOf) => {
|
||||
const blocks = new Map();
|
||||
for (const spec of OUI_REGISTRIES) {
|
||||
const entries = parseRegistry(fs.readFileSync(pathOf(spec.file), 'utf8'), spec);
|
||||
if (entries.length < spec.min) {
|
||||
throw new Error(`${spec.registry} has only ${entries.length} assignments (expected ≥ ${spec.min})`);
|
||||
}
|
||||
for (const [hex, block] of entries) blocks.set(hex, block);
|
||||
}
|
||||
return blocks;
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Lookup */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const NA = 'N/A';
|
||||
const pairs = (hex) => hex.match(/.{1,2}/g).join(':');
|
||||
|
||||
/**
|
||||
* Bits the first octet carries, valid for any query of at least two hex
|
||||
* digits. `isRand` is a likely-randomized address (Wi-Fi privacy, VMs):
|
||||
* locally administered unicast, the only mark randomization leaves on an
|
||||
* address. describeMac clears it for a registered Company ID's range.
|
||||
*/
|
||||
export const macFlags = (hex) => {
|
||||
const firstOctet = parseInt(hex.slice(0, 2), 16);
|
||||
const isMulticast = (firstOctet & 0x01) === 0x01;
|
||||
const isLocal = (firstOctet & 0x02) === 0x02;
|
||||
return {
|
||||
isMulticast,
|
||||
isUnicast: !isMulticast,
|
||||
isLocal,
|
||||
isGlobal: !isLocal,
|
||||
isRand: isLocal && !isMulticast,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* The /api/macchecker answer for normalized hex `hex` (see
|
||||
* common/mac-input.js) against `blocks`: the longest assignment that
|
||||
* covers it, its address range, and the first-octet flags. The group bit is
|
||||
* not part of an assignment, so a multicast address (01:00:5E:…) resolves
|
||||
* to the block it derives from (00:00:5E, IANA), shown as that block's
|
||||
* multicast range (01:00:5E:00:00:00–01:00:5E:FF:FF:FF). An address no
|
||||
* block covers is still answered (`found: false`) — its flags are the point
|
||||
* for a randomized one.
|
||||
*/
|
||||
export const describeMac = (blocks, hex) => {
|
||||
const firstOctet = (parseInt(hex.slice(0, 2), 16) & 0xfe).toString(16).toUpperCase().padStart(2, '0');
|
||||
const key = firstOctet + hex.slice(2);
|
||||
let assignment = null;
|
||||
for (const length of HEX_LENGTHS) {
|
||||
if (key.length >= length && blocks.has(key.slice(0, length))) {
|
||||
assignment = key.slice(0, length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
const block = assignment ? blocks.get(assignment) : null;
|
||||
const flags = macFlags(hex);
|
||||
if (!block) {
|
||||
return {
|
||||
success: true, found: false, macPrefix: pairs(hex.slice(0, 6)),
|
||||
company: NA, address: NA, country: NA,
|
||||
blockStart: NA, blockEnd: NA, blockSize: NA, blockType: NA,
|
||||
isPrivate: false, ...flags,
|
||||
};
|
||||
}
|
||||
// Shown in the query's own form: a multicast query gets its block's
|
||||
// multicast range, which holds the address asked about.
|
||||
const shown = hex.slice(0, 2) + assignment.slice(2);
|
||||
const hostDigits = MAC_HEX_LENGTH - assignment.length;
|
||||
return {
|
||||
success: true,
|
||||
found: true,
|
||||
macPrefix: pairs(shown),
|
||||
company: block.company || NA,
|
||||
address: block.address || NA,
|
||||
country: block.country || NA,
|
||||
blockStart: pairs(shown + '0'.repeat(hostDigits)),
|
||||
blockEnd: pairs(shown + 'F'.repeat(hostDigits)),
|
||||
blockSize: 16 ** hostDigits,
|
||||
blockType: block.registry,
|
||||
isPrivate: block.company.toLowerCase() === 'private',
|
||||
...flags,
|
||||
// A CID range is assigned local space (IEEE 802c ELI), not random.
|
||||
isRand: flags.isRand && block.registry !== 'CID',
|
||||
};
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Load */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
let blocks = null;
|
||||
|
||||
const loadDatabase = () => {
|
||||
if (!OUI_FILES.every((file) => fs.existsSync(path.join(OUI_DB_DIR, file)))) {
|
||||
// Info, not warn: a first boot downloads them right after.
|
||||
logger.info({ dir: OUI_DB_DIR }, '⚠️ IEEE MAC registries not present; MAC Lookup waits for the download');
|
||||
blocks = null;
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
try {
|
||||
blocks = readOuiRegistries((file) => path.join(OUI_DB_DIR, file));
|
||||
logger.info(`📦 IEEE MAC registries loaded — ${blocks.size} assignments in ${Date.now() - start}ms`);
|
||||
} catch (error) {
|
||||
// A previously loaded copy keeps serving.
|
||||
logger.warn({ err: error, dir: OUI_DB_DIR }, '⚠️ Failed to parse IEEE MAC registries');
|
||||
}
|
||||
};
|
||||
|
||||
loadDatabase();
|
||||
|
||||
/** Reload the registries after the dataset updater publishes fresh files. */
|
||||
export const reloadOuiDatabase = (reason = 'reload') => {
|
||||
logger.info(`🔄 Reloading IEEE MAC registries (${reason})`);
|
||||
loadDatabase();
|
||||
};
|
||||
|
||||
/** Whether all five registries are loaded. */
|
||||
export const isOuiLoaded = () => blocks !== null;
|
||||
|
||||
/** describeMac against the loaded registries; null when none are loaded. */
|
||||
export const lookupMac = (hex) => (blocks ? describeMac(blocks, hex) : null);
|
||||
@@ -0,0 +1,247 @@
|
||||
// Local PeeringDB lookup: a network's self-reported peering profile (types,
|
||||
// website, scope, traffic, ratio, policy, IRR as-set), the exchanges it is
|
||||
// on and the facilities it is in. server/datasets/datasets.js downloads CAIDA's
|
||||
// daily mirror of the full PeeringDB dump and distills it
|
||||
// (server/datasets/peeringdb-distill.js → buildPeeringdbIndex below); only the
|
||||
// compact per-ASN index ever lands in this directory. Contact data (poc,
|
||||
// org addresses) is never read.
|
||||
//
|
||||
// Index file (JSON):
|
||||
// { v: 1,
|
||||
// ix: { <ix id>: [name, city, country] },
|
||||
// fac: { <fac id>: [name, city, country] },
|
||||
// nets: { <asn>: { website, types[], scope, traffic, ratio, policy, irr,
|
||||
// ix: [[ix id, Mbps summed over ports, rs peer 1|0]],
|
||||
// fac: [fac id] } } }
|
||||
// Empty / "Not Disclosed" fields are omitted; a network with nothing left
|
||||
// is dropped. IX and facility rows are stored once and referenced by id.
|
||||
//
|
||||
// Sources: https://publicdata.caida.org/datasets/peeringdb/ (CAIDA AUA) ·
|
||||
// https://www.peeringdb.com
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import logger from '../logger.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const PEERINGDB_DB_DIR = path.join(__dirname, '../../common/peeringdb-db');
|
||||
export const PEERINGDB_FILE = 'peeringdb-index.json';
|
||||
export const PEERINGDB_INDEX_VERSION = 1;
|
||||
|
||||
// The four dump tables the index is built from.
|
||||
export const PEERINGDB_TABLES = ['net', 'ix', 'netixlan', 'netfac'];
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Distill: dump rows → index */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const UNDISCLOSED = 'not disclosed';
|
||||
|
||||
// Trimmed text, or undefined for empty / "Not Disclosed".
|
||||
export const cleanText = (value) => {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
const text = value.trim();
|
||||
return text && text.toLowerCase() !== UNDISCLOSED ? text : undefined;
|
||||
};
|
||||
|
||||
// http(s) URLs only, and none with userinfo — "name@host" is how an
|
||||
// e-mail address typed into the field would otherwise pass as a link.
|
||||
const cleanWebsite = (value) => {
|
||||
const text = cleanText(value);
|
||||
if (!text) return undefined;
|
||||
try {
|
||||
const url = new URL(text);
|
||||
const web = url.protocol === 'http:' || url.protocol === 'https:';
|
||||
return web && !url.username && !url.password ? text : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const isLive = (row) => !row.status || row.status === 'ok';
|
||||
|
||||
// Per-table projection to the fields the index needs, applied to each row
|
||||
// as it is scanned so nothing else is kept.
|
||||
export const projectRow = {
|
||||
net: (r) => ({
|
||||
id: r.id,
|
||||
asn: Number(r.asn),
|
||||
website: cleanWebsite(r.website),
|
||||
types: Array.isArray(r.info_types) ? r.info_types.map(cleanText).filter(Boolean) : [],
|
||||
scope: cleanText(r.info_scope),
|
||||
traffic: cleanText(r.info_traffic),
|
||||
ratio: cleanText(r.info_ratio),
|
||||
policy: cleanText(r.policy_general),
|
||||
irr: cleanText(r.irr_as_set),
|
||||
live: isLive(r),
|
||||
}),
|
||||
ix: (r) => ({ id: r.id, name: cleanText(r.name), city: cleanText(r.city), country: cleanText(r.country), live: isLive(r) }),
|
||||
netixlan: (r) => ({
|
||||
asn: Number(r.asn),
|
||||
ixId: r.ix_id,
|
||||
speed: Number(r.speed) || 0,
|
||||
rs: r.is_rs_peer === true,
|
||||
live: isLive(r) && r.operational !== false,
|
||||
}),
|
||||
netfac: (r) => ({
|
||||
netId: r.net_id,
|
||||
facId: r.fac_id,
|
||||
name: cleanText(r.name),
|
||||
city: cleanText(r.city),
|
||||
country: cleanText(r.country),
|
||||
live: isLive(r),
|
||||
}),
|
||||
};
|
||||
|
||||
const place = (row) => [row.name, row.city || '', row.country || ''];
|
||||
|
||||
/**
|
||||
* Projected rows of the four tables → index object (shape in the header).
|
||||
* Pure; the streaming distiller and the tests share it.
|
||||
*/
|
||||
export const buildPeeringdbIndex = ({ net = [], ix = [], netixlan = [], netfac = [] }) => {
|
||||
const ixById = new Map(ix.filter((r) => r.live && r.name).map((r) => [r.id, r]));
|
||||
const asnByNetId = new Map(net.map((r) => [r.id, r.asn]));
|
||||
const ixTable = {};
|
||||
const facTable = {};
|
||||
|
||||
// asn → Map<ix id, [ix id, speed, rs]>; several ports at one IX sum.
|
||||
const portsByAsn = new Map();
|
||||
for (const row of netixlan) {
|
||||
if (!row.live || !row.asn) continue;
|
||||
const exchange = ixById.get(row.ixId);
|
||||
if (!exchange) continue;
|
||||
let atIx = portsByAsn.get(row.asn);
|
||||
if (!atIx) portsByAsn.set(row.asn, (atIx = new Map()));
|
||||
const entry = atIx.get(exchange.id);
|
||||
if (entry) {
|
||||
entry[1] += row.speed;
|
||||
entry[2] = entry[2] || (row.rs ? 1 : 0);
|
||||
} else {
|
||||
atIx.set(exchange.id, [exchange.id, row.speed, row.rs ? 1 : 0]);
|
||||
ixTable[exchange.id] ??= place(exchange);
|
||||
}
|
||||
}
|
||||
|
||||
// netfac carries the facility's own name / city / country.
|
||||
const facsByAsn = new Map();
|
||||
for (const row of netfac) {
|
||||
if (!row.live || !row.name) continue;
|
||||
const asn = asnByNetId.get(row.netId);
|
||||
if (!asn) continue;
|
||||
let ids = facsByAsn.get(asn);
|
||||
if (!ids) facsByAsn.set(asn, (ids = new Set()));
|
||||
ids.add(row.facId);
|
||||
facTable[row.facId] ??= place(row);
|
||||
}
|
||||
|
||||
const byPlace = (a, b) => a[2].localeCompare(b[2]) || a[1].localeCompare(b[1]) || a[0].localeCompare(b[0]);
|
||||
const nets = {};
|
||||
for (const row of net) {
|
||||
if (!row.live || !row.asn || nets[row.asn]) continue;
|
||||
const entry = {};
|
||||
for (const key of ['website', 'scope', 'traffic', 'ratio', 'policy', 'irr']) {
|
||||
if (row[key]) entry[key] = row[key];
|
||||
}
|
||||
if (row.types.length) entry.types = row.types;
|
||||
const ports = [...(portsByAsn.get(row.asn)?.values() || [])]
|
||||
.sort((a, b) => b[1] - a[1] || ixTable[a[0]][0].localeCompare(ixTable[b[0]][0]));
|
||||
if (ports.length) entry.ix = ports;
|
||||
const facs = [...(facsByAsn.get(row.asn) || [])].sort((a, b) => byPlace(facTable[a], facTable[b]));
|
||||
if (facs.length) entry.fac = facs;
|
||||
if (Object.keys(entry).length) nets[row.asn] = entry;
|
||||
}
|
||||
|
||||
// Keep only the IX / facility rows some network still references.
|
||||
const used = (table, key) => {
|
||||
const ids = new Set();
|
||||
for (const entry of Object.values(nets)) {
|
||||
for (const item of entry[key] || []) ids.add(key === 'ix' ? item[0] : item);
|
||||
}
|
||||
return Object.fromEntries(Object.entries(table).filter(([id]) => ids.has(Number(id))));
|
||||
};
|
||||
return { v: PEERINGDB_INDEX_VERSION, ix: used(ixTable, 'ix'), fac: used(facTable, 'fac'), nets };
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Load + lookup */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const toPlaceMap = (table) => new Map(Object.entries(table || {}).map(([id, [name, city, country]]) => [
|
||||
Number(id), { name, city: city || null, country: country || null },
|
||||
]));
|
||||
|
||||
/**
|
||||
* Read an index file into lookup tables { nets, ix, fac } (Maps keyed by
|
||||
* number). The file text and the parsed object stay local, so only the
|
||||
* Maps outlive the call. Throws on a malformed or wrong-version file.
|
||||
*/
|
||||
export const readPeeringdbIndex = (filePath) => {
|
||||
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
||||
if (parsed?.v !== PEERINGDB_INDEX_VERSION || !parsed.nets) {
|
||||
throw new Error(`unsupported PeeringDB index (v=${parsed?.v})`);
|
||||
}
|
||||
const nets = new Map();
|
||||
for (const asn of Object.keys(parsed.nets)) nets.set(Number(asn), parsed.nets[asn]);
|
||||
return { nets, ix: toPlaceMap(parsed.ix), fac: toPlaceMap(parsed.fac) };
|
||||
};
|
||||
|
||||
/** One network's record from loaded tables, expanded for the API; null when absent. */
|
||||
export const expandNet = (tables, asn) => {
|
||||
const entry = tables?.nets.get(Number(asn));
|
||||
if (!entry) return null;
|
||||
return {
|
||||
website: entry.website || null,
|
||||
types: entry.types || [],
|
||||
scope: entry.scope || null,
|
||||
traffic: entry.traffic || null,
|
||||
ratio: entry.ratio || null,
|
||||
policy: entry.policy || null,
|
||||
irrAsSet: entry.irr || null,
|
||||
ixs: (entry.ix || []).flatMap(([id, speed, rs]) => {
|
||||
const exchange = tables.ix.get(id);
|
||||
return exchange ? [{ ...exchange, speed, rsPeer: rs === 1 }] : [];
|
||||
}),
|
||||
facilities: (entry.fac || []).flatMap((id) => {
|
||||
const facility = tables.fac.get(id);
|
||||
return facility ? [{ ...facility }] : [];
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
let tables = null;
|
||||
let loadedFrom = null;
|
||||
|
||||
const loadDatabase = () => {
|
||||
const filePath = path.join(PEERINGDB_DB_DIR, PEERINGDB_FILE);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
// Info, not warn: deployments without a Cloudflare key never fetch one.
|
||||
logger.info({ dir: PEERINGDB_DB_DIR }, '⚠️ PeeringDB index not present; ASN Profile shows no peering data');
|
||||
tables = null;
|
||||
loadedFrom = null;
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
try {
|
||||
tables = readPeeringdbIndex(filePath);
|
||||
loadedFrom = PEERINGDB_FILE;
|
||||
logger.info(`📦 PeeringDB index loaded — ${tables.nets.size} networks, ${tables.ix.size} exchanges, ${tables.fac.size} facilities in ${Date.now() - start}ms`);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error, path: filePath }, '⚠️ Failed to parse PeeringDB index');
|
||||
}
|
||||
};
|
||||
|
||||
loadDatabase();
|
||||
|
||||
/** Reload the index after the dataset updater publishes a fresh one. */
|
||||
export const reloadPeeringdbDatabase = (reason = 'reload') => {
|
||||
logger.info(`🔄 Reloading PeeringDB index (${reason})`);
|
||||
loadDatabase();
|
||||
};
|
||||
|
||||
/** Whether an index is loaded (false until the updater has built one). */
|
||||
export const isPeeringdbLoaded = () => loadedFrom !== null;
|
||||
|
||||
/** The network's PeeringDB record (see expandNet), or null. */
|
||||
export const lookupPeeringdb = (asn) => expandNet(tables, asn);
|
||||
@@ -0,0 +1,174 @@
|
||||
// Streaming distiller: full PeeringDB dump (CAIDA mirror, ~116 MB JSON) →
|
||||
// the compact per-ASN index server/datasets/peeringdb-db.js serves. Built for small
|
||||
// instances: the dump is never parsed whole. A byte-level scanner walks the
|
||||
// dump's fixed envelope
|
||||
// { "<table>": { "data": [ {row}, {row}, … ], "meta": {…} }, … }
|
||||
// and JSON.parses one row at a time, only inside the wanted tables' "data"
|
||||
// arrays, projecting it to the needed fields at once; every other table
|
||||
// (poc, org, fac, …) is skipped byte by byte and never materialized. The
|
||||
// index is written in batches rather than as one string.
|
||||
//
|
||||
// Scanning raw bytes is UTF-8 safe: continuation bytes (0x80–0xBF) never
|
||||
// equal '"', '\', ':', ',' or a bracket. Reads are async, one reused 1 MB
|
||||
// buffer at a time, so a running server keeps serving between chunks.
|
||||
|
||||
import fsp from 'fs/promises';
|
||||
import { PEERINGDB_TABLES, projectRow, buildPeeringdbIndex } from './peeringdb-db.js';
|
||||
|
||||
const QUOTE = 0x22;
|
||||
const BACKSLASH = 0x5c;
|
||||
const COLON = 0x3a;
|
||||
const COMMA = 0x2c;
|
||||
const OPEN_OBJECT = 0x7b;
|
||||
const CLOSE_OBJECT = 0x7d;
|
||||
const OPEN_ARRAY = 0x5b;
|
||||
const CLOSE_ARRAY = 0x5d;
|
||||
|
||||
const CHUNK_BYTES = 1 << 20;
|
||||
const WRITE_BATCH_CHARS = 1 << 20;
|
||||
|
||||
/**
|
||||
* Stream `filePath` and call `onRow(table, row)` for every row object in
|
||||
* the "data" array of each table named in `wanted`. Resolves the number of
|
||||
* rows seen; throws on a truncated document or when `signal` aborts.
|
||||
*/
|
||||
export const scanDumpRows = async (filePath, wanted, onRow, { signal } = {}) => {
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
// Container kind per depth ('o' / 'a') and whether the next string at
|
||||
// depth 1 or 2 is a key; only those two levels name tables / fields.
|
||||
const kinds = [];
|
||||
let expectKey = false;
|
||||
let keyParts = null;
|
||||
let keyStart = -1;
|
||||
let topKey = null;
|
||||
let tableKey = null;
|
||||
let rowParts = null;
|
||||
let rowStart = -1;
|
||||
let rows = 0;
|
||||
let sawDocument = false;
|
||||
|
||||
const buf = Buffer.allocUnsafe(CHUNK_BYTES);
|
||||
const handle = await fsp.open(filePath, 'r');
|
||||
try {
|
||||
for (;;) {
|
||||
signal?.throwIfAborted();
|
||||
const { bytesRead } = await handle.read(buf, 0, buf.length, null);
|
||||
if (bytesRead === 0) break;
|
||||
const chunk = buf.subarray(0, bytesRead);
|
||||
for (let i = 0; i < chunk.length; i++) {
|
||||
const b = chunk[i];
|
||||
if (inString) {
|
||||
if (escaped) escaped = false;
|
||||
else if (b === BACKSLASH) escaped = true;
|
||||
else if (b === QUOTE) {
|
||||
inString = false;
|
||||
if (keyParts) {
|
||||
keyParts.push(chunk.subarray(keyStart, i));
|
||||
const key = Buffer.concat(keyParts).toString('utf8');
|
||||
if (depth === 1) {
|
||||
topKey = key;
|
||||
tableKey = null;
|
||||
} else {
|
||||
tableKey = key;
|
||||
}
|
||||
keyParts = null;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (b === QUOTE) {
|
||||
inString = true;
|
||||
if ((depth === 1 || depth === 2) && kinds[depth] === 'o' && expectKey) {
|
||||
keyParts = [];
|
||||
keyStart = i + 1;
|
||||
}
|
||||
} else if (b === COLON) {
|
||||
if (depth <= 2) expectKey = false;
|
||||
} else if (b === COMMA) {
|
||||
if (depth <= 2) expectKey = kinds[depth] === 'o';
|
||||
} else if (b === OPEN_OBJECT || b === OPEN_ARRAY) {
|
||||
depth++;
|
||||
sawDocument = true;
|
||||
if (depth <= 3) kinds[depth] = b === OPEN_OBJECT ? 'o' : 'a';
|
||||
if (depth <= 2) expectKey = b === OPEN_OBJECT;
|
||||
if (depth === 4 && b === OPEN_OBJECT && kinds[3] === 'a'
|
||||
&& tableKey === 'data' && wanted.has(topKey)) {
|
||||
rowParts = [];
|
||||
rowStart = i;
|
||||
}
|
||||
} else if (b === CLOSE_OBJECT || b === CLOSE_ARRAY) {
|
||||
if (depth === 4 && rowParts) {
|
||||
const text = rowParts.length
|
||||
? Buffer.concat([...rowParts, chunk.subarray(rowStart, i + 1)]).toString('utf8')
|
||||
: chunk.toString('utf8', rowStart, i + 1);
|
||||
onRow(topKey, JSON.parse(text));
|
||||
rows++;
|
||||
rowParts = null;
|
||||
}
|
||||
depth--;
|
||||
}
|
||||
}
|
||||
// A row or key straddling the boundary outlives the reused buffer.
|
||||
if (rowParts) {
|
||||
rowParts.push(Buffer.from(chunk.subarray(rowStart)));
|
||||
rowStart = 0;
|
||||
}
|
||||
if (keyParts) {
|
||||
keyParts.push(Buffer.from(chunk.subarray(keyStart)));
|
||||
keyStart = 0;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
if (!sawDocument || depth !== 0 || inString) {
|
||||
throw new Error(`truncated PeeringDB dump (depth ${depth})`);
|
||||
}
|
||||
return rows;
|
||||
};
|
||||
|
||||
/** Write an index object as JSON in ~1 MB batches; same bytes as JSON.stringify. */
|
||||
export const writeIndexFile = async (outPath, index) => {
|
||||
const handle = await fsp.open(outPath, 'w');
|
||||
try {
|
||||
let pending = `{"v":${JSON.stringify(index.v)},"ix":${JSON.stringify(index.ix)},"fac":${JSON.stringify(index.fac)},"nets":{`;
|
||||
let first = true;
|
||||
for (const asn of Object.keys(index.nets)) {
|
||||
pending += `${first ? '' : ','}${JSON.stringify(asn)}:${JSON.stringify(index.nets[asn])}`;
|
||||
first = false;
|
||||
if (pending.length >= WRITE_BATCH_CHARS) {
|
||||
await handle.write(pending);
|
||||
pending = '';
|
||||
}
|
||||
}
|
||||
await handle.write(`${pending}}}`);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Dump file → index file. Resolves counts for the log; throws when the dump
|
||||
* lacks one of the four tables (a schema change must not publish).
|
||||
*/
|
||||
export const distillPeeringdbDump = async (dumpPath, outPath, { signal } = {}) => {
|
||||
const wanted = new Set(PEERINGDB_TABLES);
|
||||
const tables = Object.fromEntries(PEERINGDB_TABLES.map((name) => [name, []]));
|
||||
const rows = await scanDumpRows(dumpPath, wanted, (table, row) => {
|
||||
tables[table].push(projectRow[table](row));
|
||||
}, { signal });
|
||||
for (const name of PEERINGDB_TABLES) {
|
||||
if (!tables[name].length) throw new Error(`PeeringDB dump has no ${name} rows`);
|
||||
}
|
||||
const index = buildPeeringdbIndex(tables);
|
||||
for (const name of PEERINGDB_TABLES) tables[name] = null; // garbage before the write
|
||||
await writeIndexFile(outPath, index);
|
||||
return {
|
||||
rows,
|
||||
networks: Object.keys(index.nets).length,
|
||||
exchanges: Object.keys(index.ix).length,
|
||||
facilities: Object.keys(index.fac).length,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
// Factory for IP-geolocation source handlers.
|
||||
//
|
||||
// Every geo source (ipinfo.io, ip-api.com, ipapi.is, ip2location.io, ip.sb)
|
||||
// shares an identical Express shell: read the (already-validated) ?ip,
|
||||
// build a source-specific URL, fetch it through fetchUpstream, normalize the
|
||||
// upstream JSON into the canonical response shape, and respond — with a
|
||||
// uniform try/catch that logs the error and returns a 500.
|
||||
//
|
||||
// makeGeoHandler captures that shell. Each source supplies only:
|
||||
// - name : short id used in the error log message
|
||||
// - buildUrl : (req) => string URL OR { url, logContext } when the
|
||||
// handler wants extra fields (e.g. lang) on the error log
|
||||
// - normalize : (json) => canonical response object
|
||||
//
|
||||
// buildUrl runs before the try block so any env-key selection it performs
|
||||
// keeps its current behavior (e.g. throwing on a missing key surfaces the
|
||||
// same way the original inline handlers did).
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
import logger from './logger.js';
|
||||
|
||||
export function makeGeoHandler({ name, buildUrl, normalize }) {
|
||||
return async (req, res) => {
|
||||
// Presence, validity and public routability guaranteed by the
|
||||
// requirePublicIP middleware — a reserved address never reaches here.
|
||||
const ipAddress = req.query.ip;
|
||||
|
||||
const built = buildUrl(req);
|
||||
const url = typeof built === 'string' ? built : built.url;
|
||||
const logContext = typeof built === 'string' ? {} : (built.logContext || {});
|
||||
|
||||
try {
|
||||
const apiRes = await fetchUpstream(url);
|
||||
// Outage / gateway pages come back as HTML — fail on status
|
||||
// instead of letting JSON.parse throw on "<html>".
|
||||
if (!apiRes.ok) {
|
||||
throw new Error(`Upstream responded ${apiRes.status}`);
|
||||
}
|
||||
const json = await apiRes.json();
|
||||
res.json(normalize(json));
|
||||
} catch (e) {
|
||||
logger.error({ err: e, ip: ipAddress, ...logContext }, `${name} handler failed`);
|
||||
res.status(500).json({ error: e.message });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Pure aggregation for the Globalping probe inventory: turns the raw
|
||||
// /v1/probes payload (one entry per online probe) into the compact shape
|
||||
// the frontend country pickers consume — which countries have a probe at
|
||||
// all, grouped into the five continent buckets the UI displays (NA + SA
|
||||
// merge into "americas"). Serving this from our backend keeps the multi-
|
||||
// hundred-KB probe list off the visitor's connection.
|
||||
|
||||
// Globalping continent codes → our display buckets.
|
||||
const CONTINENT_BUCKETS = { AS: 'asia', EU: 'europe', AF: 'africa', NA: 'americas', SA: 'americas', OC: 'oceania' };
|
||||
const BUCKET_ORDER = ['asia', 'europe', 'africa', 'americas', 'oceania'];
|
||||
|
||||
// probes: raw array from Globalping. Returns
|
||||
// { countries: [cc], continents: [{ key, countries: [cc] }] } with
|
||||
// continents in display order and country codes sorted alphabetically.
|
||||
export const buildProbeInventory = (probes) => {
|
||||
const countries = new Set();
|
||||
const buckets = new Map();
|
||||
|
||||
for (const probe of (Array.isArray(probes) ? probes : [])) {
|
||||
const cc = probe?.location?.country;
|
||||
if (!cc) continue;
|
||||
countries.add(cc);
|
||||
const bucket = CONTINENT_BUCKETS[probe?.location?.continent];
|
||||
if (!bucket) continue;
|
||||
if (!buckets.has(bucket)) buckets.set(bucket, new Set());
|
||||
buckets.get(bucket).add(cc);
|
||||
}
|
||||
|
||||
return {
|
||||
countries: [...countries].sort(),
|
||||
continents: BUCKET_ORDER
|
||||
.filter((key) => buckets.has(key))
|
||||
.map((key) => ({ key, countries: [...buckets.get(key)].sort() })),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,164 @@
|
||||
// Express middleware that factors out the boilerplate every api/ handler
|
||||
// used to repeat at the top of its function body: referer check, param
|
||||
// presence check, param validity check. Mount once in backend-server.js;
|
||||
// handlers stop carrying the defensive checks and can't accidentally forget
|
||||
// them.
|
||||
|
||||
import { refererCheck } from './referer-check.js';
|
||||
import { isValidIP, isValidDomain, isUsablePublicIP } from '../common/valid-ip.js';
|
||||
import { isValidBgpPrefix } from '../common/bgp-prefix.js';
|
||||
import { STATUS_PROVIDER_IDS } from './service-status-providers.js';
|
||||
import { DNS_RECORD_TYPE_SET } from '../common/dns-record-types.js';
|
||||
import { parseAsnInput } from '../common/asn-input.js';
|
||||
|
||||
// Reject requests without an allowed referer. The error message variant
|
||||
// preserves the existing user-facing wording.
|
||||
export const requireReferer = (req, res, next) => {
|
||||
const referer = req.headers.referer;
|
||||
if (!refererCheck(referer)) {
|
||||
return res.status(403).json({
|
||||
error: referer ? 'Access denied' : 'What are you doing?',
|
||||
});
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests whose IP query param isn't a publicly routable address.
|
||||
// Every `?ip=` route asks an external service about that address — a
|
||||
// registry, a geolocation source — and reserved space has no answer there,
|
||||
// so it never reaches an upstream call. Malformed input and reserved space
|
||||
// get distinct messages; the two failures need different fixes.
|
||||
// Factory so a route using another param name (none today, but leaving the
|
||||
// door open) can say `requirePublicIP('target')`.
|
||||
export const requirePublicIP = (paramName = 'ip') => (req, res, next) => {
|
||||
const ip = req.query[paramName];
|
||||
if (!ip) {
|
||||
return res.status(400).json({ error: 'No IP address provided' });
|
||||
}
|
||||
if (!isValidIP(ip)) {
|
||||
return res.status(400).json({ error: 'Invalid IP address' });
|
||||
}
|
||||
if (!isUsablePublicIP(ip)) {
|
||||
return res.status(400).json({ error: 'Not a public IP address' });
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests without a syntactically valid domain in the specified
|
||||
// query param. Lowercases the value in place so handlers and the edge cache
|
||||
// see one canonical form (domains are case-insensitive; a mixed-case query
|
||||
// must not become a distinct CF cache entry).
|
||||
export const requireValidDomain = (paramName = 'domain') => (req, res, next) => {
|
||||
const raw = req.query[paramName];
|
||||
if (!raw) {
|
||||
return res.status(400).json({ error: 'No domain provided' });
|
||||
}
|
||||
const domain = String(raw).toLowerCase();
|
||||
if (!isValidDomain(domain)) {
|
||||
return res.status(400).json({ error: 'Invalid domain' });
|
||||
}
|
||||
req.query[paramName] = domain;
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests without a valid CIDR prefix in the specified query param.
|
||||
// Accepts any well-formed CIDR — the quantization policy (e.g. /24 for v4,
|
||||
// /48 for v6) is the frontend's job, not the guard's.
|
||||
export const requireValidPrefix = (paramName = 'prefix') => (req, res, next) => {
|
||||
const prefix = req.query[paramName];
|
||||
if (!prefix) {
|
||||
return res.status(400).json({ error: 'No prefix provided' });
|
||||
}
|
||||
if (!isValidBgpPrefix(prefix)) {
|
||||
return res.status(400).json({ error: 'Invalid prefix' });
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests without a valid ASN (AS1 … AS4294967295, up to ten
|
||||
// digits, optional 'AS' prefix — parseAsnInput in common/asn-input.js) and
|
||||
// rewrite it in place to the canonical number string (`AS013335` → `13335`),
|
||||
// so the edge cache sees one key. Used by /api/asn-connectivity,
|
||||
// /api/asn-profile and, per-view, the /api/cfradar dispatcher.
|
||||
export const requireValidASN = (paramName = 'asn') => (req, res, next) => {
|
||||
const raw = req.query[paramName];
|
||||
if (!raw) {
|
||||
return res.status(400).json({ error: 'No ASN provided' });
|
||||
}
|
||||
const asn = parseAsnInput(raw);
|
||||
if (asn === null) {
|
||||
return res.status(400).json({ error: 'Invalid ASN' });
|
||||
}
|
||||
req.query[paramName] = String(asn);
|
||||
next();
|
||||
};
|
||||
|
||||
// Canonicalize an ASN-shaped value (`AS13335`, `as13335` — the `AS` prefix
|
||||
// is required) to `AS<n>` in place, rejecting AS0 and anything past 32 bits.
|
||||
// Any other value, a bare number included, passes through untouched —
|
||||
// /api/whois also takes IPs and domains, which its handler validates.
|
||||
export const normalizeAsnQuery = (paramName = 'q') => (req, res, next) => {
|
||||
const raw = req.query[paramName];
|
||||
if (typeof raw !== 'string' || !/^AS\d+$/i.test(raw.trim())) return next();
|
||||
const asn = parseAsnInput(raw, { requirePrefix: true });
|
||||
if (asn === null) {
|
||||
return res.status(400).json({ error: 'Invalid ASN' });
|
||||
}
|
||||
req.query[paramName] = `AS${asn}`;
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests without a two-letter country code; uppercases in place so
|
||||
// the edge cache sees one canonical key. Syntactic only — an unassigned code
|
||||
// just yields an empty upstream series.
|
||||
export const requireValidCountry = (paramName = 'country') => (req, res, next) => {
|
||||
const raw = req.query[paramName];
|
||||
if (!raw) {
|
||||
return res.status(400).json({ error: 'No country provided' });
|
||||
}
|
||||
if (!/^[A-Za-z]{2}$/.test(raw)) {
|
||||
return res.status(400).json({ error: 'Invalid country' });
|
||||
}
|
||||
req.query[paramName] = String(raw).toUpperCase();
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests whose report id doesn't look like one we issued: 16 random
|
||||
// bytes as base64url, always exactly 22 chars. Reads the ROUTE param (not the
|
||||
// query) — the share-report read endpoint is /api/report/:id.
|
||||
export const requireValidReportId = (paramName = 'id') => (req, res, next) => {
|
||||
const id = req.params?.[paramName];
|
||||
if (!id || !/^[A-Za-z0-9_-]{22}$/.test(id)) {
|
||||
return res.status(400).json({ error: 'Invalid report id' });
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
// Whitelist ?type= against the record types the resolver actually handles.
|
||||
// Without it the DoH branch forwards any string verbatim to four third-party
|
||||
// endpoints, which makes this route a query proxy for types we never support.
|
||||
export const requireValidRecordType = (paramName = 'type') => (req, res, next) => {
|
||||
const type = req.query[paramName];
|
||||
if (!type) {
|
||||
return res.status(400).json({ error: 'No record type provided' });
|
||||
}
|
||||
if (!DNS_RECORD_TYPE_SET.has(String(type).toUpperCase())) {
|
||||
return res.status(400).json({ error: 'Invalid record type' });
|
||||
}
|
||||
req.query[paramName] = String(type).toUpperCase();
|
||||
next();
|
||||
};
|
||||
|
||||
// Reject requests whose `id` isn't a known service-status provider slug.
|
||||
// Used by the per-provider components / incidents endpoints, which select a
|
||||
// row from the in-memory snapshot by id.
|
||||
export const requireValidProviderId = (paramName = 'id') => (req, res, next) => {
|
||||
const id = req.query[paramName];
|
||||
if (!id) {
|
||||
return res.status(400).json({ error: 'No provider id provided' });
|
||||
}
|
||||
if (!STATUS_PROVIDER_IDS.has(id)) {
|
||||
return res.status(400).json({ error: 'Invalid provider id' });
|
||||
}
|
||||
next();
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
// Derives the IANA timezone of an IP-geolocation result from the coordinates
|
||||
// that same result reports, so the timezone can never contradict the city and
|
||||
// country rendered beside it — a card reading "Los Angeles / Asia/Tokyo"
|
||||
// because a second database disagreed about where the IP lives.
|
||||
//
|
||||
// This is the timezone *of a looked-up IP*. The visitor's own browser timezone
|
||||
// is a separate concern living in frontend/utils/time-utils.js.
|
||||
//
|
||||
// Only the zone name travels to the client. The UTC offset is rendered
|
||||
// frontend-side from this name: geo routes sit behind a 24h edge cache, and a
|
||||
// cached offset would be an hour wrong for everyone from a DST switch until
|
||||
// the entry expired.
|
||||
|
||||
import tzlookup from '@photostructure/tz-lookup';
|
||||
|
||||
// Sources spell a missing coordinate as 'N/A', null, '' or omit it. Number()
|
||||
// turns null and '' into 0, which is a real point in the Gulf of Guinea, so
|
||||
// coercion alone would invent a timezone — accept numbers and numeric strings
|
||||
// only, and reject everything else outright.
|
||||
const toCoordinate = (value) => {
|
||||
if (typeof value === 'number') {
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
if (typeof value === 'string' && value.trim() !== '') {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// `Etc/*` is what the lookup returns for open water. Reaching it means the
|
||||
// source's coordinates are unusable, not that the IP sits in a UTC-fixed zone.
|
||||
const isOpenWaterZone = (zone) => zone.startsWith('Etc/');
|
||||
|
||||
/**
|
||||
* Resolve coordinates to an IANA zone name, or '' when they can't be trusted.
|
||||
*/
|
||||
export const lookupTimeZone = (latitude, longitude) => {
|
||||
const lat = toCoordinate(latitude);
|
||||
const lon = toCoordinate(longitude);
|
||||
|
||||
if (lat === null || lon === null) {
|
||||
return '';
|
||||
}
|
||||
|
||||
try {
|
||||
const zone = tzlookup(lat, lon);
|
||||
return isOpenWaterZone(zone) ? '' : zone;
|
||||
} catch {
|
||||
// Out-of-range coordinates; the source's position data is broken.
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Add `timezone` to a geo payload in place, always defined so the frontend has
|
||||
* one falsy check rather than a presence check. Payloads without usable
|
||||
* coordinates get '' — those responses carry no city either, so the card stays
|
||||
* consistent by omitting both.
|
||||
*/
|
||||
export const attachTimeZone = (payload) => {
|
||||
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
payload.timezone = lookupTimeZone(payload.latitude, payload.longitude);
|
||||
return payload;
|
||||
};
|
||||
|
||||
/**
|
||||
* Route middleware that enriches a geo handler's response. Hooks res.json the
|
||||
* same way cacheable() does, and on the same 2xx condition, so error bodies
|
||||
* never grow the field. Handlers stay unaware of it; a new geo source inherits
|
||||
* the field by adding this middleware to its route in backend-server.js.
|
||||
*/
|
||||
export const withTimeZone = () => (req, res, next) => {
|
||||
const originalJson = res.json.bind(res);
|
||||
res.json = function (body) {
|
||||
if (res.statusCode < 400) {
|
||||
attachTimeZone(body);
|
||||
}
|
||||
return originalJson(body);
|
||||
};
|
||||
next();
|
||||
};
|
||||
@@ -0,0 +1,114 @@
|
||||
// server/logger.js — shared pino logger for all backend code.
|
||||
//
|
||||
// LOG_LEVEL defaults to 'warn'; LOG_FORMAT=json switches off pino-pretty
|
||||
// for log shippers. No NODE_ENV dependency.
|
||||
//
|
||||
// When SENTRY_DSN_BACKEND is set, the logMethod hook below mirrors log lines
|
||||
// to Sentry (no official pino transport exists; the hook runs in-process and
|
||||
// needs no extra dependency), split by severity to match the frontend's
|
||||
// signal philosophy:
|
||||
// - warn+ → Sentry Logs (queryable stream, Explore → Logs)
|
||||
// - error+ → ALSO a Sentry Issue (grouped by the `err` stack when present,
|
||||
// alertable) — a caught-and-logged upstream failure is a signal,
|
||||
// not just a log line
|
||||
// Without the DSN, @sentry/node is never loaded and the hook is not
|
||||
// installed at all — the usual gating philosophy.
|
||||
//
|
||||
// ES module imports are hoisted, so backend-server.js's dotenv.config()
|
||||
// runs after this file — we load .env ourselves here so LOG_LEVEL is
|
||||
// honored. dotenv is idempotent; `quiet: true` avoids double banners.
|
||||
|
||||
import dotenv from 'dotenv';
|
||||
import pino from 'pino';
|
||||
|
||||
dotenv.config({ quiet: true });
|
||||
|
||||
const useJson = process.env.LOG_FORMAT === 'json';
|
||||
|
||||
let sentry = null;
|
||||
if (process.env.SENTRY_DSN_BACKEND) {
|
||||
sentry = await import('@sentry/node');
|
||||
}
|
||||
|
||||
// Flatten a pino context object into Sentry log attributes, which must be
|
||||
// primitives: Errors become message + stack strings, nested objects become
|
||||
// JSON. Exported for tests.
|
||||
export const flattenLogAttributes = (ctx) => {
|
||||
const attributes = {};
|
||||
for (const [key, value] of Object.entries(ctx)) {
|
||||
if (value instanceof Error) {
|
||||
attributes[`${key}.message`] = value.message;
|
||||
attributes[`${key}.stack`] = value.stack ?? '';
|
||||
} else if (typeof value === 'object' && value !== null) {
|
||||
try {
|
||||
attributes[key] = JSON.stringify(value);
|
||||
} catch {
|
||||
attributes[key] = String(value);
|
||||
}
|
||||
} else {
|
||||
attributes[key] = value;
|
||||
}
|
||||
}
|
||||
return attributes;
|
||||
};
|
||||
|
||||
// pino numeric level → Sentry log level; anything below warn stays local.
|
||||
const SENTRY_LOG_LEVELS = { 40: 'warn', 50: 'error', 60: 'fatal' };
|
||||
|
||||
const logger = pino({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
...(sentry ? {
|
||||
hooks: {
|
||||
// Mirror warn+ lines to Sentry Logs and elevate error+ to Issues,
|
||||
// then log normally. pino call shapes are (msg) or (ctx, msg).
|
||||
// Only enabled levels reach this hook, so LOG_LEVEL filtering
|
||||
// applies to Sentry too.
|
||||
logMethod(args, method, level) {
|
||||
const sentryLevel = SENTRY_LOG_LEVELS[level];
|
||||
if (sentryLevel) {
|
||||
try {
|
||||
const [first, second] = args;
|
||||
const msg = String(typeof first === 'string' ? first : (second ?? ''));
|
||||
const ctx = (typeof first === 'object' && first !== null) ? first : {};
|
||||
const attributes = flattenLogAttributes(ctx);
|
||||
sentry.logger[sentryLevel](msg, attributes);
|
||||
if (level >= 50) {
|
||||
// Group by the original throw site when an err
|
||||
// object is attached (the logger.error({ err }...)
|
||||
// convention); the log message rides along as a
|
||||
// searchable tag.
|
||||
const captureContext = {
|
||||
level: sentryLevel,
|
||||
tags: { log_message: msg },
|
||||
extra: attributes,
|
||||
};
|
||||
if (ctx.err instanceof Error) {
|
||||
sentry.captureException(ctx.err, captureContext);
|
||||
} else {
|
||||
sentry.captureMessage(msg, captureContext);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Telemetry must never break local logging.
|
||||
}
|
||||
}
|
||||
return method.apply(this, args);
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
...(useJson ? {} : {
|
||||
transport: {
|
||||
target: 'pino-pretty',
|
||||
options: {
|
||||
colorize: true,
|
||||
// SYS: = host-local time; 'standard' carries date + UTC offset
|
||||
// (same rendering as the MyIP-api repo's logger)
|
||||
translateTime: 'SYS:standard',
|
||||
ignore: 'pid,hostname',
|
||||
singleLine: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
export default logger;
|
||||
@@ -0,0 +1,42 @@
|
||||
// Boot-time readiness of the offline datasets (server/datasets/datasets.js) and
|
||||
// the primed service-status snapshot. The backend listens before any of them
|
||||
// is downloaded; runOfflineBootstrap marks that window, and requireOfflineData
|
||||
// answers 503 on a route whose data is still missing inside it. Outside the
|
||||
// window a missing dataset is a failed download, served degraded as before.
|
||||
// A response assembled from independent parts (the ASN Profile) instead asks
|
||||
// isStillLoading per part and leaves just that part out, uncached.
|
||||
//
|
||||
// Snapshots already on disk are loaded before the listener opens (CAIDA,
|
||||
// PeeringDB and IEEE at import, MaxMind in backend-server.js), so a restart
|
||||
// never hits the gate — only a first boot or a newly added dataset does.
|
||||
|
||||
let booting = false;
|
||||
|
||||
// Run the boot steps in parallel, gating until all settle. Steps log their
|
||||
// own failures; one that throws still lets the gate go.
|
||||
export const runOfflineBootstrap = async (steps) => {
|
||||
booting = true;
|
||||
try {
|
||||
await Promise.allSettled(steps.map((step) => Promise.resolve().then(step)));
|
||||
} finally {
|
||||
booting = false;
|
||||
}
|
||||
};
|
||||
|
||||
// Whether the boot window is open: some offline data may still be on its way.
|
||||
export const isOfflineBootstrapping = () => booting;
|
||||
|
||||
// Whether any of `checks` is unready inside the boot window — data that is
|
||||
// on its way, as opposed to a download that already failed.
|
||||
export const isStillLoading = (...checks) => booting && checks.some((isReady) => !isReady());
|
||||
|
||||
// `checks` are the route's `() => boolean` readiness probes (isMaxMindReady,
|
||||
// isAsRelLoaded, …). 503 is never edge-cached (cacheable only stamps < 400),
|
||||
// so nothing degraded is pinned.
|
||||
export const requireOfflineData = (checks) => (req, res, next) => {
|
||||
if (isStillLoading(...checks)) {
|
||||
res.setHeader('Retry-After', '30');
|
||||
return res.status(503).json({ error: 'Offline data is loading' });
|
||||
}
|
||||
next();
|
||||
};
|
||||
@@ -0,0 +1,83 @@
|
||||
// Pure classification logic for the OONI-backed censorship view: turns raw
|
||||
// rows from OONI's aggregation API (probe_cc × blocking_type, see
|
||||
// api/ooni-blocking.js) into one tiered entry per country.
|
||||
//
|
||||
// Tier semantics — calibrated against real 30-day data for known-blocked
|
||||
// domains (twitter/facebook/bbc/pornhub, 2026-07):
|
||||
// confirmed OONI fingerprint-confirmed blocking (blockpage served). Only
|
||||
// exists in countries that serve identifiable blockpages.
|
||||
// likely High anomaly share. Countries that block via DNS poisoning /
|
||||
// RST injection (e.g. CN) never produce `confirmed`, so anomaly
|
||||
// share is the primary signal, not a fallback.
|
||||
// signs Moderate anomaly share — intermittent interference or noisy
|
||||
// CDN geo-variance. Requires a larger sample to avoid false
|
||||
// positives from low-traffic countries.
|
||||
// ok Everything else, including under-sampled countries.
|
||||
|
||||
export const OONI_WINDOW_DAYS = 30;
|
||||
|
||||
// Minimum measurements for a country to be classifiable at all. Kept low on
|
||||
// purpose: for rarely-tested domains a handful of measurements is all OONI
|
||||
// has, and 7-of-8 anomalous is still a real signal (the ratio thresholds do
|
||||
// the actual gatekeeping). Below this, one noisy probe would decide the tier.
|
||||
const MIN_SAMPLES = 5;
|
||||
// `signs` is the noisiest tier; demand a larger sample before flagging.
|
||||
const SIGNS_MIN_SAMPLES = 30;
|
||||
|
||||
const CONFIRMED_RATIO = 0.05;
|
||||
const LIKELY_RATIO = 0.5;
|
||||
const SIGNS_RATIO = 0.2;
|
||||
|
||||
const TIER_ORDER = { confirmed: 0, likely: 1, signs: 2, ok: 3 };
|
||||
|
||||
// OONI's blocking_type values for web_connectivity anomalies. Rows with an
|
||||
// empty blocking_type carry the ok / failure counts and never contribute to
|
||||
// the per-method breakdown.
|
||||
const BLOCKING_TYPES = new Set(['dns', 'tcp_ip', 'http-failure', 'http-diff']);
|
||||
|
||||
const classifyTier = ({ measurements, anomaly, confirmed }) => {
|
||||
if (measurements < MIN_SAMPLES) return 'ok';
|
||||
const blockedRatio = (anomaly + confirmed) / measurements;
|
||||
if (confirmed / measurements >= CONFIRMED_RATIO) return 'confirmed';
|
||||
if (blockedRatio >= LIKELY_RATIO) return 'likely';
|
||||
if (measurements >= SIGNS_MIN_SAMPLES && blockedRatio >= SIGNS_RATIO) return 'signs';
|
||||
return 'ok';
|
||||
};
|
||||
|
||||
// rows: concatenated OONI result rows (both hostname variants are fine — the
|
||||
// accumulator merges by probe_cc). Returns one entry per country, flagged
|
||||
// tiers first, each tier sorted by blocked share descending.
|
||||
export const classifyOoniCountries = (rows) => {
|
||||
const byCountry = new Map();
|
||||
|
||||
for (const row of rows) {
|
||||
const cc = row?.probe_cc;
|
||||
if (!cc) continue;
|
||||
let entry = byCountry.get(cc);
|
||||
if (!entry) {
|
||||
entry = { country: cc, measurements: 0, ok: 0, anomaly: 0, confirmed: 0, failure: 0, methods: {} };
|
||||
byCountry.set(cc, entry);
|
||||
}
|
||||
entry.measurements += row.measurement_count || 0;
|
||||
entry.ok += row.ok_count || 0;
|
||||
entry.anomaly += row.anomaly_count || 0;
|
||||
entry.confirmed += row.confirmed_count || 0;
|
||||
entry.failure += row.failure_count || 0;
|
||||
|
||||
const bt = row.blocking_type;
|
||||
if (BLOCKING_TYPES.has(bt)) {
|
||||
const hits = (row.anomaly_count || 0) + (row.confirmed_count || 0);
|
||||
if (hits > 0) entry.methods[bt] = (entry.methods[bt] || 0) + hits;
|
||||
}
|
||||
}
|
||||
|
||||
const blockedShare = (e) => e.measurements ? (e.anomaly + e.confirmed) / e.measurements : 0;
|
||||
|
||||
return [...byCountry.values()]
|
||||
.map((e) => ({ ...e, tier: classifyTier(e) }))
|
||||
.sort((a, b) =>
|
||||
(TIER_ORDER[a.tier] - TIER_ORDER[b.tier])
|
||||
|| (blockedShare(b) - blockedShare(a))
|
||||
|| (b.measurements - a.measurements)
|
||||
);
|
||||
};
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
// RDAP lookups for the WHOIS tool.
|
||||
//
|
||||
// - Domains: fallback. whoiser covers legacy gTLDs well through
|
||||
// port 43, but newer ones (.ing / .app / .dev / …) expose RDAP only
|
||||
// and return no WHOIS at all — rdapDomain() fills that gap.
|
||||
// - IPs: primary. RIR RDAP is plain HTTPS + JSON from the
|
||||
// authoritative registry, with none of port-43's referral quirks
|
||||
// (rwhois:// endpoints speak a protocol WHOIS clients can't follow).
|
||||
// - ASNs: only source. RIR autnum objects, same bootstrap mechanism.
|
||||
//
|
||||
// Public API:
|
||||
// rdapDomain(name) → { [host]: { __raw, ...rdapJson } }
|
||||
// Same outer shape as whoiser.domain() so the handler can splice
|
||||
// the result in without any frontend change.
|
||||
// rdapIp(ip) → { __raw, ...rdapJson }
|
||||
// Flat, like whoiser.ip() — the frontend only reads `__raw`.
|
||||
// rdapAutnum(asn) → { asn, handle, name, rir, …, __raw }
|
||||
// Flat like rdapIp, but parsed fields instead of the raw RDAP JSON.
|
||||
//
|
||||
// Bootstrap (IANA's TLD / address-space → RDAP endpoint maps) is cached
|
||||
// in-memory for 24h per file. Upstream calls go through `fetchUpstream`
|
||||
// so they inherit the project's timeout convention.
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
import { isIPv6 } from '../common/valid-ip.js';
|
||||
import { ipToBigInt, parseCidr, prefixContains } from '../common/ip-math.js';
|
||||
import logger from './logger.js';
|
||||
|
||||
const BOOTSTRAP_BASE = 'https://data.iana.org/rdap/';
|
||||
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const bootstrapCache = new Map(); // file → { data, expiresAt }
|
||||
|
||||
const loadBootstrap = async (file) => {
|
||||
const cached = bootstrapCache.get(file);
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
return cached.data;
|
||||
}
|
||||
const res = await fetchUpstream(BOOTSTRAP_BASE + file);
|
||||
if (!res.ok) {
|
||||
logger.error({ file, status: res.status }, 'RDAP bootstrap failed');
|
||||
throw new Error(`RDAP bootstrap failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
bootstrapCache.set(file, { data, expiresAt: Date.now() + CACHE_TTL_MS });
|
||||
return data;
|
||||
};
|
||||
|
||||
function findEndpoint(services, tld) {
|
||||
const needle = tld.toLowerCase();
|
||||
for (const [tlds, urls] of services) {
|
||||
if (tlds.some(t => t.toLowerCase() === needle)) return urls[0];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimSlash = (u) => u.replace(/\/$/, '');
|
||||
|
||||
export async function rdapDomain(domain, { timeoutMs = 5000 } = {}) {
|
||||
const bootstrap = await loadBootstrap('dns.json');
|
||||
const tld = domain.split('.').pop();
|
||||
const base = findEndpoint(bootstrap.services, tld);
|
||||
if (!base) {
|
||||
throw new Error(`No RDAP endpoint for .${tld}`);
|
||||
}
|
||||
|
||||
const host = new URL(base).hostname;
|
||||
const url = `${trimSlash(base)}/domain/${encodeURIComponent(domain)}`;
|
||||
const res = await fetchUpstream(url, { timeoutMs });
|
||||
if (res.status === 404) {
|
||||
throw new Error(`Domain not found: ${domain}`);
|
||||
}
|
||||
if (!res.ok) {
|
||||
logger.error({ domain, status: res.status }, 'RDAP query failed');
|
||||
throw new Error(`RDAP query failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
|
||||
return { [host]: { ...data, __raw: formatDomain(data) } };
|
||||
}
|
||||
|
||||
// -- IP lookup -------------------------------------------------------------
|
||||
|
||||
// Longest-prefix match of `ip` against an IANA ipv4/ipv6 bootstrap
|
||||
// `services` array (entries: [[cidr, …], [url, …]]). Exported for tests.
|
||||
export const findIpEndpoint = (services, ip) => {
|
||||
const v6 = isIPv6(ip);
|
||||
const ipBig = ipToBigInt(ip);
|
||||
if (ipBig === null) return null;
|
||||
|
||||
let best = null;
|
||||
let bestLen = -1;
|
||||
for (const [cidrs, urls] of services) {
|
||||
for (const cidr of cidrs) {
|
||||
const block = parseCidr(cidr);
|
||||
if (!block || (block.family === 6) !== v6) continue;
|
||||
if (block.prefix <= bestLen
|
||||
|| !prefixContains(block.network, block.prefix, block.family, ipBig)) continue;
|
||||
bestLen = block.prefix;
|
||||
best = urls.find((u) => u.startsWith('https://')) || urls[0];
|
||||
}
|
||||
}
|
||||
return best;
|
||||
};
|
||||
|
||||
export const rdapIp = async (ip, { timeoutMs = 5000 } = {}) => {
|
||||
const bootstrap = await loadBootstrap(isIPv6(ip) ? 'ipv6.json' : 'ipv4.json');
|
||||
const base = findIpEndpoint(bootstrap.services, ip);
|
||||
if (!base) {
|
||||
throw new Error(`No RDAP endpoint for ${ip}`);
|
||||
}
|
||||
|
||||
// Literal IP, not encodeURIComponent: colons are legal in a URL path,
|
||||
// the guard layer already ensures a well-formed IP, and some RIR
|
||||
// delegates (e.g. IDNIC behind APNIC's redirect) reject %3A with a 400.
|
||||
const url = `${trimSlash(base)}/ip/${ip}`;
|
||||
const res = await fetchUpstream(url, { timeoutMs });
|
||||
if (res.status === 404) {
|
||||
throw new Error(`IP not found: ${ip}`);
|
||||
}
|
||||
if (!res.ok) {
|
||||
logger.error({ ip, status: res.status }, 'RDAP IP query failed');
|
||||
throw new Error(`RDAP IP query failed: ${res.status}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
|
||||
return { ...data, __raw: formatIpNetwork(data) };
|
||||
};
|
||||
|
||||
// -- Autnum lookup ---------------------------------------------------------
|
||||
|
||||
// Match an ASN against IANA asn.json bootstrap services (entries:
|
||||
// [[range, …], [url, …]], ranges "N" or "N-M"). Exported for tests.
|
||||
export const findAutnumEndpoint = (services, asn) => {
|
||||
for (const [ranges, urls] of services) {
|
||||
for (const range of ranges) {
|
||||
const [lo, hi] = range.split('-').map(Number);
|
||||
if (asn >= lo && asn <= (hi ?? lo)) {
|
||||
return urls.find((u) => u.startsWith('https://')) || urls[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// RDAP service host → registry display name. Unlisted hosts (an NIR, a
|
||||
// moved endpoint) fall back to the hostname itself.
|
||||
const RIR_BY_HOST = {
|
||||
'rdap.arin.net': 'ARIN',
|
||||
'rdap.db.ripe.net': 'RIPE NCC',
|
||||
'rdap.apnic.net': 'APNIC',
|
||||
'rdap.lacnic.net': 'LACNIC',
|
||||
'rdap.afrinic.net': 'AFRINIC',
|
||||
};
|
||||
|
||||
// Depth-first walk over an entity tree (RDAP nests contacts inside their
|
||||
// parent entity), returning the first entity carrying `role`.
|
||||
const findEntityByRole = (entities, role) => {
|
||||
for (const e of entities || []) {
|
||||
if (e.roles?.includes(role)) return e;
|
||||
const nested = findEntityByRole(e.entities, role);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const vcardOrgName = (v) => (Array.isArray(v.org?.[0]) ? v.org[0].join(' ') : v.org?.[0]);
|
||||
|
||||
// RDAP autnum document → the parsed fields plus the WHOIS-like `__raw`
|
||||
// block. `host` is the RDAP service that answered. Exported for tests.
|
||||
export const parseAutnum = (data, asn, host) => {
|
||||
const events = {};
|
||||
for (const e of data.events || []) events[e.eventAction] = e.eventDate;
|
||||
const registrant = findEntityByRole(data.entities, 'registrant');
|
||||
const registrantCard = registrant ? extractVcard(registrant) : {};
|
||||
const abuseCard = extractVcard(findEntityByRole(data.entities, 'abuse'));
|
||||
return {
|
||||
asn,
|
||||
handle: data.handle || null,
|
||||
name: data.name || null,
|
||||
rir: RIR_BY_HOST[host] || host || null,
|
||||
status: data.status || [],
|
||||
registered: events.registration || null,
|
||||
lastChanged: events['last changed'] || null,
|
||||
registrant: registrantCard.fn?.[0] || vcardOrgName(registrantCard) || null,
|
||||
country: data.country || null,
|
||||
abuse: abuseCard.email?.[0] || null,
|
||||
__raw: formatAutnum(data),
|
||||
};
|
||||
};
|
||||
|
||||
// True for rdapAutnum's "no record" failures (unregistered ASN, no RDAP
|
||||
// service for its range) as opposed to an upstream fault.
|
||||
export const isAutnumMissing = (error) => /^(ASN not found|No RDAP endpoint for )/.test(error?.message || '');
|
||||
|
||||
export const rdapAutnum = async (asn, { timeoutMs = 5000 } = {}) => {
|
||||
const n = Number(String(asn).replace(/^AS/i, ''));
|
||||
const bootstrap = await loadBootstrap('asn.json');
|
||||
const base = findAutnumEndpoint(bootstrap.services, n);
|
||||
if (!base) {
|
||||
throw new Error(`No RDAP endpoint for AS${n}`);
|
||||
}
|
||||
|
||||
const url = `${trimSlash(base)}/autnum/${n}`;
|
||||
const res = await fetchUpstream(url, { timeoutMs });
|
||||
if (res.status === 404) {
|
||||
throw new Error(`ASN not found: AS${n}`);
|
||||
}
|
||||
if (!res.ok) {
|
||||
logger.error({ asn: n, status: res.status }, 'RDAP autnum query failed');
|
||||
throw new Error(`RDAP autnum query failed: ${res.status}`);
|
||||
}
|
||||
return parseAutnum(await res.json(), n, new URL(base).hostname);
|
||||
};
|
||||
|
||||
// -- Format RDAP JSON into a WHOIS-like text block ------------------------
|
||||
|
||||
function extractVcard(entity) {
|
||||
const props = entity?.vcardArray?.[1] || [];
|
||||
const out = {};
|
||||
for (const [name, , , value] of props) {
|
||||
if (name === 'version') continue;
|
||||
if (!out[name]) out[name] = [];
|
||||
out[name].push(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function formatEntity(entity, indent) {
|
||||
const lines = [];
|
||||
const pad = ' '.repeat(indent);
|
||||
if (entity.handle) lines.push(`${pad}Handle: ${entity.handle}`);
|
||||
const v = extractVcard(entity);
|
||||
if (v.fn) lines.push(`${pad}Name: ${v.fn[0]}`);
|
||||
if (v.org) lines.push(`${pad}Org: ${Array.isArray(v.org[0]) ? v.org[0].join(' ') : v.org[0]}`);
|
||||
if (v.email) lines.push(`${pad}Email: ${v.email.join(', ')}`);
|
||||
if (v.tel) lines.push(`${pad}Phone: ${v.tel.join(', ')}`);
|
||||
if (v.adr) {
|
||||
const addr = v.adr[0];
|
||||
if (Array.isArray(addr)) {
|
||||
const s = addr.filter(Boolean).join(', ');
|
||||
if (s) lines.push(`${pad}Address: ${s}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// Group entities by role and render one section per entity, `order`
|
||||
// first, leftover roles after. RDAP nests contacts inside their parent
|
||||
// entity (e.g. abuse/tech inside an ARIN org) — the walk flattens those
|
||||
// so nested contacts still surface.
|
||||
const formatEntitySections = (entities, order) => {
|
||||
const byRole = new Map();
|
||||
const collect = (list) => {
|
||||
for (const e of list || []) {
|
||||
for (const role of (e.roles?.length ? e.roles : ['unknown'])) {
|
||||
if (!byRole.has(role)) byRole.set(role, []);
|
||||
byRole.get(role).push(e);
|
||||
}
|
||||
collect(e.entities);
|
||||
}
|
||||
};
|
||||
collect(entities);
|
||||
|
||||
const lines = [];
|
||||
const seen = new Set();
|
||||
for (const role of order) {
|
||||
for (const e of byRole.get(role) || []) {
|
||||
seen.add(e);
|
||||
lines.push('');
|
||||
lines.push(`${role[0].toUpperCase()}${role.slice(1)}:`);
|
||||
lines.push(...formatEntity(e, 2));
|
||||
}
|
||||
}
|
||||
for (const [role, es] of byRole) {
|
||||
if (order.includes(role)) continue;
|
||||
for (const e of es) {
|
||||
if (seen.has(e)) continue;
|
||||
lines.push('');
|
||||
lines.push(`${role}:`);
|
||||
lines.push(...formatEntity(e, 2));
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
};
|
||||
|
||||
// WHOIS-like text block for an RDAP IP-network object, field names
|
||||
// mirroring what RIR port-43 output calls them. Exported for tests.
|
||||
export const formatIpNetwork = (data) => {
|
||||
const lines = [];
|
||||
lines.push(`NetRange: ${data.startAddress || 'N/A'} - ${data.endAddress || 'N/A'}`);
|
||||
|
||||
const cidrs = (data.cidr0_cidrs || [])
|
||||
.map((c) => `${c.v4prefix || c.v6prefix}/${c.length}`)
|
||||
.filter((c) => !c.startsWith('undefined'));
|
||||
if (cidrs.length) lines.push(`CIDR: ${cidrs.join(', ')}`);
|
||||
|
||||
if (data.name) lines.push(`NetName: ${data.name}`);
|
||||
if (data.handle) lines.push(`Handle: ${data.handle}`);
|
||||
if (data.parentHandle) lines.push(`Parent: ${data.parentHandle}`);
|
||||
if (data.type) lines.push(`NetType: ${data.type}`);
|
||||
if (data.country) lines.push(`Country: ${data.country}`);
|
||||
|
||||
const ev = {};
|
||||
for (const e of data.events || []) ev[e.eventAction] = e.eventDate;
|
||||
if (ev.registration) lines.push(`Created: ${ev.registration}`);
|
||||
if (ev['last changed']) lines.push(`Updated: ${ev['last changed']}`);
|
||||
|
||||
if (data.status?.length) {
|
||||
lines.push('Status:');
|
||||
for (const s of data.status) lines.push(` ${s}`);
|
||||
}
|
||||
|
||||
lines.push(...formatEntitySections(
|
||||
data.entities,
|
||||
['registrant', 'administrative', 'technical', 'abuse', 'noc', 'routing'],
|
||||
));
|
||||
|
||||
for (const remark of data.remarks || []) {
|
||||
lines.push('');
|
||||
lines.push(`${remark.title || 'Remarks'}:`);
|
||||
for (const d of remark.description || []) lines.push(` ${d}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
// WHOIS-like text block for an RDAP autnum object, field names mirroring
|
||||
// RIR port-43 aut-num output. Exported for tests.
|
||||
export const formatAutnum = (data) => {
|
||||
const lines = [];
|
||||
const start = data.startAutnum;
|
||||
const end = data.endAutnum;
|
||||
if (start != null) {
|
||||
lines.push(`ASNumber: ${end != null && end !== start ? `${start} - ${end}` : start}`);
|
||||
}
|
||||
if (data.name) lines.push(`ASName: ${data.name}`);
|
||||
if (data.handle) lines.push(`ASHandle: ${data.handle}`);
|
||||
if (data.type) lines.push(`Type: ${data.type}`);
|
||||
if (data.country) lines.push(`Country: ${data.country}`);
|
||||
|
||||
const ev = {};
|
||||
for (const e of data.events || []) ev[e.eventAction] = e.eventDate;
|
||||
if (ev.registration) lines.push(`Created: ${ev.registration}`);
|
||||
if (ev['last changed']) lines.push(`Updated: ${ev['last changed']}`);
|
||||
|
||||
if (data.status?.length) {
|
||||
lines.push('Status:');
|
||||
for (const s of data.status) lines.push(` ${s}`);
|
||||
}
|
||||
|
||||
lines.push(...formatEntitySections(
|
||||
data.entities,
|
||||
['registrant', 'administrative', 'technical', 'abuse', 'noc', 'routing'],
|
||||
));
|
||||
|
||||
for (const remark of data.remarks || []) {
|
||||
lines.push('');
|
||||
lines.push(`${remark.title || 'Remarks'}:`);
|
||||
for (const d of remark.description || []) lines.push(` ${d}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
function formatDomain(data) {
|
||||
const lines = [];
|
||||
lines.push(`Domain Name: ${data.ldhName || 'N/A'}`);
|
||||
if (data.unicodeName && data.unicodeName !== data.ldhName) {
|
||||
lines.push(`Unicode Name: ${data.unicodeName}`);
|
||||
}
|
||||
if (data.handle) lines.push(`Registry Domain ID: ${data.handle}`);
|
||||
|
||||
const ev = {};
|
||||
for (const e of data.events || []) ev[e.eventAction] = e.eventDate;
|
||||
if (ev.registration) lines.push(`Created: ${ev.registration}`);
|
||||
if (ev['last changed']) lines.push(`Updated: ${ev['last changed']}`);
|
||||
if (ev.expiration) lines.push(`Expires: ${ev.expiration}`);
|
||||
if (ev['last update of RDAP database']) lines.push(`RDAP Last Refresh: ${ev['last update of RDAP database']}`);
|
||||
|
||||
if (data.status?.length) {
|
||||
lines.push('Status:');
|
||||
for (const s of data.status) lines.push(` ${s}`);
|
||||
}
|
||||
|
||||
lines.push(...formatEntitySections(
|
||||
data.entities,
|
||||
['registrar', 'registrant', 'administrative', 'technical', 'abuse', 'reseller'],
|
||||
));
|
||||
|
||||
if (data.nameservers?.length) {
|
||||
lines.push('');
|
||||
lines.push('Name Servers:');
|
||||
for (const ns of data.nameservers) lines.push(` ${ns.ldhName || 'N/A'}`);
|
||||
}
|
||||
if (data.secureDNS) {
|
||||
lines.push('');
|
||||
lines.push(`DNSSEC: ${data.secureDNS.delegationSigned ? 'signed' : 'unsigned'}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// dotenv.config() is called once in backend-server.js before any handler
|
||||
// imports this module, so process.env.ALLOWED_DOMAINS is already populated.
|
||||
// Avoid the duplicate call to keep this a pure, fast function.
|
||||
|
||||
function refererCheck(referer) {
|
||||
const allowedDomains = ['localhost', ...(process.env.ALLOWED_DOMAINS || '').split(',')];
|
||||
|
||||
if (referer) {
|
||||
// Scanners send garbage Referer headers; a parse failure means
|
||||
// "not an allowed origin", never a thrown 500.
|
||||
let domain;
|
||||
try {
|
||||
domain = new URL(referer).hostname;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return allowedDomains.includes(domain);
|
||||
}
|
||||
return false; // if no referer is provided, return false
|
||||
}
|
||||
|
||||
export { refererCheck };
|
||||
@@ -0,0 +1,65 @@
|
||||
// Centralized RIPEstat client. Each typed helper picks a timeout matching
|
||||
// its endpoint's normal latency; callers can override per call.
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
import { lookupAsOrgName } from './datasets/as-org-db.js';
|
||||
|
||||
const BASE_URL = 'https://stat.ripe.net/data';
|
||||
const SOURCE_APP = process.env.RIPESTAT_SOURCE_APP || 'myip';
|
||||
|
||||
function fetchRipestat(endpoint, resource, { timeoutMs = 8000, params = {} } = {}) {
|
||||
const search = new URLSearchParams({ resource, sourceapp: SOURCE_APP, ...params });
|
||||
return fetchUpstream(`${BASE_URL}/${endpoint}/data.json?${search}`, { timeoutMs });
|
||||
}
|
||||
|
||||
/** as-overview: AS metadata including `holder` (org name w/ handle prefix). */
|
||||
export function fetchAsOverview(asn, { timeoutMs = 3000 } = {}) {
|
||||
return fetchRipestat('as-overview', `AS${asn}`, { timeoutMs });
|
||||
}
|
||||
|
||||
/** routing-history: historical AS announcements for a prefix or IP.
|
||||
* Slow analytical endpoint — scans years of BGP data, regularly takes
|
||||
* 10–20s for prefixes with long history.
|
||||
*
|
||||
* Pass `minPeersSeeing` to push the caller's peer floor down to RIPEstat:
|
||||
* rows seen by fewer RIS peers are route noise the caller would discard
|
||||
* anyway, so dropping them during the scan trims the response without
|
||||
* changing the result. The threshold is owned by the caller (asn-history's
|
||||
* MIN_PEERS) — omit it and RIPEstat keeps its own default. */
|
||||
export function fetchRoutingHistory(resource, { timeoutMs = 25000, minPeersSeeing } = {}) {
|
||||
const params = minPeersSeeing != null ? { min_peers_seeing: minPeersSeeing } : {};
|
||||
return fetchRipestat('routing-history', resource, { timeoutMs, params });
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip RIPEstat's "<HANDLE> - " prefix from a `holder` field so the UI
|
||||
* sees just the readable company name.
|
||||
*/
|
||||
export function extractOrgFromHolder(holder) {
|
||||
if (!holder || typeof holder !== 'string') return null;
|
||||
const dash = holder.indexOf(' - ');
|
||||
return dash > 0 ? holder.slice(dash + 3).trim() : holder.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-tier AS org-name resolver: local CAIDA as2org first (µs), RIPEstat
|
||||
* as-overview fallback when the snapshot doesn't have the ASN. Best-effort —
|
||||
* any failure yields null so callers can fall back to ASN-only display.
|
||||
*
|
||||
* `onError(error, asn)` is an optional observability hook invoked on the
|
||||
* fallback failure/catch path. asn-history passes a warn logger here;
|
||||
* asn-connectivity omits it and stays silent — preserve that asymmetry.
|
||||
*/
|
||||
export async function resolveAsnOrgName(asn, { onError } = {}) {
|
||||
const local = lookupAsOrgName(asn);
|
||||
if (local) return local;
|
||||
try {
|
||||
const res = await fetchAsOverview(asn);
|
||||
if (!res.ok) return null;
|
||||
const payload = await res.json();
|
||||
return extractOrgFromHolder(payload?.data?.holder);
|
||||
} catch (error) {
|
||||
if (onError) onError(error, asn);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Sentry Cron check-in wrapper for the backend's periodic jobs (the dataset
|
||||
// updater, service-status). Same gating as sentry-instrument.js: without
|
||||
// SENTRY_DSN_BACKEND this is a pure passthrough and @sentry/node is never
|
||||
// loaded, so job modules can call it unconditionally.
|
||||
//
|
||||
// The monitor is upserted from the monitorConfig on first check-in — nothing
|
||||
// to pre-create in the Sentry UI. A rejected job promise reports an "error"
|
||||
// check-in and rethrows, so callers keep their own .catch logging.
|
||||
|
||||
let sentryWithMonitor = null;
|
||||
// Check-ins are production-only: a dev machine's monitor starts reporting
|
||||
// "missed" the moment the dev server stops, paging about a closed laptop.
|
||||
// Everything else about Sentry still works normally in development.
|
||||
if (process.env.SENTRY_DSN_BACKEND && process.env.SENTRY_ENVIRONMENT !== 'development') {
|
||||
({ withMonitor: sentryWithMonitor } = await import('@sentry/node'));
|
||||
}
|
||||
|
||||
export const withCronMonitor = (slug, fn, monitorConfig) => {
|
||||
if (!sentryWithMonitor) return fn();
|
||||
return sentryWithMonitor(slug, fn, monitorConfig);
|
||||
};
|
||||
@@ -0,0 +1,81 @@
|
||||
// Scrubbing helpers for backend Sentry telemetry. Query strings on
|
||||
// upstream calls carry API keys — redact those param values everywhere
|
||||
// (breadcrumbs, spans, event request contexts) before sending. The rest
|
||||
// of the query is kept on purpose: it is debugging context. Wired into
|
||||
// Sentry.init hooks in sentry-instrument.js;
|
||||
// kept here as pure functions so they stay testable without loading the SDK.
|
||||
|
||||
// Param names whose values must never reach Sentry.
|
||||
const SENSITIVE_PARAMS = /^(key|api[-_]?key|token|secret|password|auth)$/i;
|
||||
|
||||
// Breadcrumb / span / trace attributes that may hold a URL, or a bare
|
||||
// query string.
|
||||
const URL_ATTRS = ['url', 'url.full', 'http.url', 'http.target'];
|
||||
const QUERY_ATTRS = ['url.query', 'http.query'];
|
||||
|
||||
// Redact sensitive values in a bare query string ("q=1&key=abc"). Node's
|
||||
// http instrumentation stores query attributes with a leading `?`
|
||||
// ("?key=abc&ip=…") — tolerate it so the first param still matches.
|
||||
export const redactQueryString = (query) => {
|
||||
if (typeof query !== 'string') return query;
|
||||
return query.split('&').map((pair) => {
|
||||
const eq = pair.indexOf('=');
|
||||
if (eq === -1) return pair;
|
||||
const rawName = pair.slice(0, eq);
|
||||
const name = rawName.charAt(0) === '?' ? rawName.slice(1) : rawName;
|
||||
return SENSITIVE_PARAMS.test(name) ? `${rawName}=[redacted]` : pair;
|
||||
}).join('&');
|
||||
};
|
||||
|
||||
// Redact sensitive query params in a full or relative URL. Anything
|
||||
// before the first `?` passes through untouched.
|
||||
export const redactUrlQuery = (value) => {
|
||||
if (typeof value !== 'string') return value;
|
||||
const q = value.indexOf('?');
|
||||
if (q === -1) return value;
|
||||
return value.slice(0, q + 1) + redactQueryString(value.slice(q + 1));
|
||||
};
|
||||
|
||||
const scrubAttributes = (data) => {
|
||||
if (!data) return;
|
||||
for (const attr of URL_ATTRS) {
|
||||
if (typeof data[attr] === 'string') data[attr] = redactUrlQuery(data[attr]);
|
||||
}
|
||||
for (const attr of QUERY_ATTRS) {
|
||||
if (typeof data[attr] === 'string') data[attr] = redactQueryString(data[attr]);
|
||||
}
|
||||
};
|
||||
|
||||
// beforeBreadcrumb: the http integration records the outgoing URL across
|
||||
// several data attributes (url, http.query, …) — scrub them all.
|
||||
export const scrubBreadcrumb = (breadcrumb) => {
|
||||
scrubAttributes(breadcrumb?.data);
|
||||
return breadcrumb;
|
||||
};
|
||||
|
||||
// beforeSendSpan: http client spans carry the URL both in attributes and
|
||||
// in the description ("GET https://host/path?query").
|
||||
export const scrubSpan = (span) => {
|
||||
if (!span) return span;
|
||||
if (typeof span.description === 'string') {
|
||||
span.description = redactUrlQuery(span.description);
|
||||
}
|
||||
scrubAttributes(span.data);
|
||||
return span;
|
||||
};
|
||||
|
||||
// beforeSend / beforeSendTransaction: inbound request context plus the
|
||||
// root span's trace attributes.
|
||||
export const scrubEventRequest = (event) => {
|
||||
if (!event) return event;
|
||||
if (event.request) {
|
||||
if (typeof event.request.url === 'string') {
|
||||
event.request.url = redactUrlQuery(event.request.url);
|
||||
}
|
||||
if (typeof event.request.query_string === 'string') {
|
||||
event.request.query_string = redactQueryString(event.request.query_string);
|
||||
}
|
||||
}
|
||||
scrubAttributes(event.contexts?.trace?.data);
|
||||
return event;
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
// Upstream source-of-truth for the "Service Status" homepage section.
|
||||
//
|
||||
// Each provider exposes a status page that is API-compatible with the Atlassian
|
||||
// Statuspage / incident.io schema — both expose the same
|
||||
// `/api/v2/summary.json` + `/api/v2/incidents.json` endpoints. Cloudflare
|
||||
// self-hosts a page that mirrors the same shape.
|
||||
//
|
||||
// `id` — stable slug
|
||||
// `name` — display name
|
||||
// `api` — origin we hit `${api}/api/v2/{summary,incidents}.json` against
|
||||
// `page` — public status page; also the base for incident detail links
|
||||
// (`${page}/incidents/${incidentId}`), since incident.io's feed omits
|
||||
// a usable shortlink.
|
||||
//
|
||||
// This list lives backend-side: the poller (server/service-status-store.js)
|
||||
// walks it on a fixed schedule and caches the result in memory, so the
|
||||
// upstream origins never reach the shipped frontend bundle.
|
||||
|
||||
export const STATUS_PROVIDERS = [
|
||||
{ id: 'claude', name: 'Claude', api: 'https://status.claude.com', page: 'https://status.claude.com' },
|
||||
{ id: 'chatgpt', name: 'ChatGPT', api: 'https://status.openai.com', page: 'https://status.openai.com' },
|
||||
{ id: 'cursor', name: 'Cursor', api: 'https://status.cursor.com', page: 'https://status.cursor.com' },
|
||||
// exclude: drop GitHub's "Visit … for more information" placeholder row.
|
||||
{ id: 'github', name: 'GitHub', api: 'https://www.githubstatus.com', page: 'https://www.githubstatus.com', components: { exclude: ['0l2p9nhqnxpd'] } },
|
||||
{ id: 'elevenlabs', name: 'ElevenLabs', api: 'https://status.elevenlabs.io', page: 'https://status.elevenlabs.io' },
|
||||
// Default top-level: summary.json flattens away CF's mid-level product
|
||||
// categories, leaving one "Cloudflare Sites and Services" rollup + 7
|
||||
// continents — which keeps the card light consistent with this list.
|
||||
{ id: 'cloudflare', name: 'Cloudflare', api: 'https://new.cloudflarestatus.com', page: 'https://new.cloudflarestatus.com' },
|
||||
{ id: 'groq', name: 'Groq', api: 'https://groqstatus.com', page: 'https://groqstatus.com' },
|
||||
{ id: 'notion', name: 'Notion', api: 'https://www.notion-status.com', page: 'https://www.notion-status.com' },
|
||||
{ id: 'vercel', name: 'Vercel', api: 'https://www.vercel-status.com', page: 'https://www.vercel-status.com' },
|
||||
{ id: 'netlify', name: 'Netlify', api: 'https://www.netlifystatus.com', page: 'https://www.netlifystatus.com' },
|
||||
{ id: 'render', name: 'Render', api: 'https://status.render.com', page: 'https://status.render.com' },
|
||||
{ id: 'supabase', name: 'Supabase', api: 'https://status.supabase.com', page: 'https://status.supabase.com' },
|
||||
{ id: 'discord', name: 'Discord', api: 'https://discordstatus.com', page: 'https://discordstatus.com' },
|
||||
{ id: 'figma', name: 'Figma', api: 'https://status.figma.com', page: 'https://status.figma.com' },
|
||||
{ id: 'linear', name: 'Linear', api: 'https://linearstatus.com', page: 'https://linearstatus.com' },
|
||||
// Stripe's customer-facing status.stripe.com is a homegrown page with a
|
||||
// non-Statuspage schema; www.stripestatus.com is its Statuspage mirror,
|
||||
// which exposes the /api/v2 endpoints this poller needs.
|
||||
{ id: 'stripe', name: 'Stripe', api: 'https://www.stripestatus.com', page: 'https://www.stripestatus.com' },
|
||||
];
|
||||
|
||||
// Whitelist used by requireValidProviderId() — the per-provider detail
|
||||
// endpoints (components / incidents) take an `id` query param.
|
||||
export const STATUS_PROVIDER_IDS = new Set(STATUS_PROVIDERS.map((p) => p.id));
|
||||
@@ -0,0 +1,125 @@
|
||||
// In-memory poller + cache for the "Service Status" section.
|
||||
//
|
||||
// Rather than fetch upstream status pages on each visitor request, a single
|
||||
// background timer refreshes all providers every REFRESH_INTERVAL and keeps
|
||||
// the latest normalized snapshot in memory. Request handlers just read the
|
||||
// snapshot — upstream load is constant (16 providers × 2 endpoints / 5 min)
|
||||
// no matter how much traffic the site gets.
|
||||
//
|
||||
// Mirrors the bootstrap-at-boot + start-scheduler shape of the MaxMind / CAIDA
|
||||
// updaters, but kept deliberately lean: memory only, no files, no locks.
|
||||
|
||||
import { fetchUpstream } from '../common/fetch-with-timeout.js';
|
||||
import logger from './logger.js';
|
||||
import { withCronMonitor } from './sentry-cron.js';
|
||||
import { STATUS_PROVIDERS } from './service-status-providers.js';
|
||||
import { assembleProvider } from './service-status-transform.js';
|
||||
|
||||
const REFRESH_INTERVAL_MS = 5 * 60 * 1000;
|
||||
|
||||
// Latest snapshot served to clients. `updatedAt` stays null until the first
|
||||
// successful-or-degraded refresh lands.
|
||||
let snapshot = { updatedAt: null, providers: [] };
|
||||
let schedulerStarted = false;
|
||||
|
||||
// Fetch + parse one JSON endpoint; throws on non-2xx so the caller can degrade.
|
||||
async function fetchJson(url) {
|
||||
const res = await fetchUpstream(url);
|
||||
if (!res.ok) throw new Error(`upstream ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// Refresh a single provider. Never throws.
|
||||
//
|
||||
// `previous` is this provider's last published entry (if any). On a failed
|
||||
// summary fetch we serve last-known-good rather than flipping a healthy
|
||||
// provider to 'unknown' — upstream status pages have frequent transient
|
||||
// blips (DNS hiccups, TLS resets) that shouldn't surface as "status
|
||||
// unavailable". Only the genuinely-degraded case (no prior good data) is a
|
||||
// warn; recoverable blips drop to debug to keep the prod log quiet.
|
||||
async function refreshProvider(provider, previous) {
|
||||
const [summary, incidents] = await Promise.allSettled([
|
||||
fetchJson(`${provider.api}/api/v2/summary.json`),
|
||||
fetchJson(`${provider.api}/api/v2/incidents.json`),
|
||||
]);
|
||||
const summaryJson = summary.status === 'fulfilled' ? summary.value : null;
|
||||
const incidentsJson = incidents.status === 'fulfilled' ? incidents.value : null;
|
||||
|
||||
if (!summaryJson) {
|
||||
const hadGood = previous && previous.indicator !== 'unknown';
|
||||
if (hadGood) {
|
||||
logger.debug({ provider: provider.id }, 'service-status refresh blip; serving last-known-good');
|
||||
return previous;
|
||||
}
|
||||
logger.warn({ err: summary.reason, provider: provider.id }, 'service-status summary fetch failed (no prior data)');
|
||||
}
|
||||
|
||||
const entry = assembleProvider(provider, summaryJson, incidentsJson);
|
||||
// Keep prior incidents if only the incidents endpoint blipped this tick.
|
||||
if (!incidentsJson && previous?.incidents?.length) {
|
||||
entry.incidents = previous.incidents;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
// Refresh every provider in parallel and publish a new snapshot.
|
||||
export async function refreshServiceStatus() {
|
||||
const prevById = new Map(snapshot.providers.map((p) => [p.id, p]));
|
||||
const providers = await Promise.all(
|
||||
STATUS_PROVIDERS.map((p) => refreshProvider(p, prevById.get(p.id))),
|
||||
);
|
||||
snapshot = { updatedAt: new Date().toISOString(), providers };
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
// Lightweight overview: every provider's status light, without the heavier
|
||||
// components / incidents arrays. Served by /api/service-status so the initial
|
||||
// page load stays small; detail is pulled per-provider on demand.
|
||||
export function getServiceStatusOverview() {
|
||||
return {
|
||||
updatedAt: snapshot.updatedAt,
|
||||
providers: snapshot.providers.map(({ id, name, page, indicator }) => ({
|
||||
id, name, page, indicator,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
// One provider's full cached entry (or null if not in the snapshot yet).
|
||||
// Backs the components / incidents detail endpoints.
|
||||
export function getProviderDetail(id) {
|
||||
return snapshot.providers.find((p) => p.id === id) || null;
|
||||
}
|
||||
|
||||
// Whether a refresh has completed — false until the boot fill lands.
|
||||
export const isServiceStatusPrimed = () => snapshot.updatedAt !== null;
|
||||
|
||||
// Populate the cache once at boot. Non-fatal: a failure leaves an empty
|
||||
// snapshot that the next scheduled tick will fill.
|
||||
export async function bootstrapServiceStatus() {
|
||||
try {
|
||||
await refreshServiceStatus();
|
||||
logger.info('📦 Service status cache primed');
|
||||
} catch (error) {
|
||||
logger.warn({ err: error }, '⚠️ Service status initial refresh failed; will retry on schedule');
|
||||
}
|
||||
}
|
||||
|
||||
// Start the 5-minute refresh loop. Idempotent; timer is unref'd so it never
|
||||
// keeps the process alive on its own.
|
||||
export function startServiceStatusPolling() {
|
||||
if (schedulerStarted) return;
|
||||
schedulerStarted = true;
|
||||
// withCronMonitor reports ok/error check-ins to Sentry Crons (no-op
|
||||
// without a backend DSN) so a dead poller surfaces as a missed check-in.
|
||||
const tick = () => {
|
||||
withCronMonitor('service-status-refresh', refreshServiceStatus, {
|
||||
schedule: { type: 'interval', value: REFRESH_INTERVAL_MS / (60 * 1000), unit: 'minute' },
|
||||
checkinMargin: 5,
|
||||
maxRuntime: 5,
|
||||
}).catch((error) => {
|
||||
logger.error({ err: error }, 'service-status refresh tick failed');
|
||||
});
|
||||
};
|
||||
setInterval(tick, REFRESH_INTERVAL_MS).unref?.();
|
||||
logger.info(`🗓️ Service status auto refresh every ${REFRESH_INTERVAL_MS / 60000} minutes (${STATUS_PROVIDERS.length} providers)`);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Pure, network-free normalizers for the Statuspage / incident.io schema.
|
||||
//
|
||||
// Both `summary.json` and `incidents.json` share the same shape across every
|
||||
// provider we track, so a single pair of transforms covers all of them. Kept
|
||||
// dependency-free and side-effect-free so they're unit-testable with inline
|
||||
// fixtures (tests/service-status-transform.test.js) — no live upstream calls.
|
||||
|
||||
// Safety cap on forwarded components — a guard against a pathological page
|
||||
// bloating the payload. Selected lists are normally well under this.
|
||||
const COMPONENT_LIMIT = 40;
|
||||
|
||||
// Pick which components a provider exposes (vendors structure pages too
|
||||
// differently for one rule to fit all). Per-provider selector — see STATUS_PROVIDERS:
|
||||
// (default) top-level rows: group headers + ungrouped components
|
||||
// { flatten: true } every leaf, group headers dropped
|
||||
// { include: [names] } ordered whitelist by component name
|
||||
// { exclude: [id|name] } final filter on top of any mode (drops junk rows)
|
||||
function selectComponents(raw, selector = {}) {
|
||||
let picked;
|
||||
if (Array.isArray(selector.include)) {
|
||||
const byName = new Map(raw.filter((c) => !c?.group).map((c) => [c?.name, c]));
|
||||
picked = selector.include.map((name) => byName.get(name)).filter(Boolean);
|
||||
} else if (selector.flatten) {
|
||||
picked = raw.filter((c) => !c?.group); // all leaves, drop group headers
|
||||
} else {
|
||||
picked = raw.filter((c) => !c?.group_id); // top-level rows
|
||||
}
|
||||
|
||||
if (Array.isArray(selector.exclude) && selector.exclude.length) {
|
||||
const blocked = new Set(selector.exclude);
|
||||
picked = picked.filter((c) => !blocked.has(c?.id) && !blocked.has(c?.name));
|
||||
}
|
||||
return picked;
|
||||
}
|
||||
|
||||
// Normalize a `/api/v2/summary.json` payload to
|
||||
// { indicator, components: [{ name, status }] }
|
||||
// where indicator is 'none' | 'minor' | 'major' | 'critical' | 'maintenance' | 'unknown'.
|
||||
// `selector` chooses which components to keep (see selectComponents).
|
||||
export function normalizeSummary(json, selector = {}) {
|
||||
const raw = Array.isArray(json?.components) ? json.components : [];
|
||||
const components = selectComponents(raw, selector)
|
||||
.slice(0, COMPONENT_LIMIT)
|
||||
.map((c) => ({ name: c?.name ?? '', status: c?.status ?? 'operational' }));
|
||||
|
||||
return {
|
||||
indicator: json?.status?.indicator || 'unknown',
|
||||
components,
|
||||
};
|
||||
}
|
||||
|
||||
// Normalize a `/api/v2/incidents.json` payload to the most recent `limit`
|
||||
// incidents. Upstream already returns them newest-first.
|
||||
//
|
||||
// `pageUrl` is the provider's status-page origin; each incident's detail link
|
||||
// is built as `${pageUrl}/incidents/${id}`. We construct it rather than trust
|
||||
// the feed's `shortlink` because incident.io pages (ChatGPT(OpenAI), Notion) omit it —
|
||||
// their feed otherwise points only at the page root, not the incident.
|
||||
//
|
||||
// Returns: [{ id, name, status, impact, startedAt, url }]
|
||||
//
|
||||
// Sorted newest-first by `startedAt` (the timestamp we display) before
|
||||
// capping, NOT by upstream order: incident.io feeds (ChatGPT(OpenAI), Notion) leave
|
||||
// `started_at` null and order by `updated_at`, so the `created_at` we fall
|
||||
// back to would otherwise render out of chronological order in the UI.
|
||||
export function normalizeIncidents(json, { limit = 10, pageUrl = '' } = {}) {
|
||||
const base = String(pageUrl).replace(/\/+$/, ''); // trim trailing slash(es)
|
||||
const incidents = Array.isArray(json?.incidents) ? json.incidents : [];
|
||||
return incidents
|
||||
.map((i) => {
|
||||
const id = i?.id ?? null;
|
||||
return {
|
||||
id,
|
||||
name: i?.name ?? '',
|
||||
status: i?.status ?? null,
|
||||
impact: i?.impact ?? 'none',
|
||||
// `started_at` is the canonical start; fall back to created_at.
|
||||
startedAt: i?.started_at ?? i?.created_at ?? null,
|
||||
url: id && base ? `${base}/incidents/${id}` : (base || null),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => incidentTimestamp(b) - incidentTimestamp(a))
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
// Parse an incident's display timestamp to a comparable number; missing /
|
||||
// unparseable dates sort oldest (0) so they fall to the bottom of the list.
|
||||
function incidentTimestamp(incident) {
|
||||
if (!incident.startedAt) return 0;
|
||||
const ms = Date.parse(incident.startedAt);
|
||||
return Number.isNaN(ms) ? 0 : ms;
|
||||
}
|
||||
|
||||
// Combine one provider's config with its (possibly null) summary + incidents
|
||||
// payloads into the shape the frontend consumes. Pure: a null payload (failed
|
||||
// fetch) degrades gracefully — normalizeSummary(null) → indicator 'unknown'
|
||||
// with no components, normalizeIncidents(null) → []. Keeps the poller's
|
||||
// per-provider error handling free of shape-building logic.
|
||||
export function assembleProvider(provider, summaryJson, incidentsJson) {
|
||||
const { indicator, components } = normalizeSummary(summaryJson, provider.components);
|
||||
return {
|
||||
id: provider.id,
|
||||
name: provider.name,
|
||||
page: provider.page,
|
||||
indicator,
|
||||
components,
|
||||
incidents: normalizeIncidents(incidentsJson, { pageUrl: provider.page }),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Backend-only User-Agent bootstrap for fetchUpstream.
|
||||
//
|
||||
// Some upstream WAFs (e.g. Cloudflare's status page) hard-block undici's
|
||||
// default `User-Agent: node`, so server-to-server calls identify themselves
|
||||
// as `MyIP/v<version>/<site-url>` instead. Version comes from package.json;
|
||||
// the site segment from VITE_SITE_URL, so forks advertise their own
|
||||
// deployment rather than ipcheck.ing. Lives outside fetch-with-timeout.js on
|
||||
// purpose: that module is shared with the browser bundle and must stay free
|
||||
// of fs / process access.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
import { setUpstreamUserAgent } from '../common/fetch-with-timeout.js';
|
||||
|
||||
// Build and register the UA. Called from backend-server.js after
|
||||
// dotenv.config() so VITE_SITE_URL from .env is visible — module import
|
||||
// time would be too early (imports are hoisted above the config call).
|
||||
// Missing pieces degrade gracefully: `MyIP/v7.1.0`, or just `MyIP`.
|
||||
export const initUpstreamUserAgent = () => {
|
||||
let version = '';
|
||||
try {
|
||||
version = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version || '';
|
||||
} catch {
|
||||
// Unreadable package.json → versionless UA.
|
||||
}
|
||||
const site = (process.env.VITE_SITE_URL || '').trim();
|
||||
const ua = ['MyIP', version && `v${version}`, site].filter(Boolean).join('/');
|
||||
setUpstreamUserAgent(ua);
|
||||
return ua;
|
||||
};
|
||||
Reference in new issue
Block a user