diff --git a/.gitignore b/.gitignore index c7460151..15165f03 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,11 @@ common/as-org-db/*.bak common/as-rel-db/*.bak common/as-org-db/*.next common/as-rel-db/*.next +common/peeringdb-db/*.json +common/peeringdb-db/.caida-update-state.json +common/peeringdb-db/.caida-update.lock +common/peeringdb-db/*.bak +common/peeringdb-db/*.next .learnings/ docs/ .plan/ diff --git a/api/AGENTS.md b/api/AGENTS.md index caaa2428..f3442683 100644 --- a/api/AGENTS.md +++ b/api/AGENTS.md @@ -32,15 +32,20 @@ behind `dns-resolver` (gated by `tests/dns-resolvers-data.test.js`). ### ASN data - **`/api/asn-profile`** (`common/asn-profile.js`) runs each ASN source's - existing core under its own deadline; a failed or late one is just absent. - Always 200 `{ asn, status, …sections }` (`ok` / `empty` / `error` / - `disabled`); 502 only if all fail. + existing core under its own deadline, set above the upstream timeouts it + wraps (`INNER_TIMEOUTS`) so a source degrading inside still answers; a failed + or late one is just absent. Always 200 `{ asn, status, …sections }` (`ok` / + `empty` / `error` / `disabled`); 502 only if all fail. - **Relationship counts follow the graph:** `asn-connectivity` neighbours and the Radar `asn` view's counts come from local CAIDA (`common/as-relationships.js`), dropping the providers the graph walk distrusts. - **`/api/whois` `?q=`:** IPs (RDAP, whoiser fallback), domains (whoiser, RDAP fallback), `AS` (RDAP autnum); all carry a `__raw` block. +- **Offline datasets** (`common/caida-updater.js` rows): fetched when missing at + boot, refreshed daily with `CAIDA_AUTO_UPDATE=true`; PeeringDB only with a + Cloudflare key. Its 116 MB dump is stream-distilled (never parsed whole) to a + ~6 MB per-ASN index without contact data (`common/peeringdb-distill.js`). ## Conventions @@ -98,8 +103,7 @@ behind `dns-resolver` (gated by `tests/dns-resolvers-data.test.js`). string to four third-party endpoints. - `requireValidReportId()` — `/api/report/:id` route param (22-char base64url). -New param shape → new guard in `common/guards.js`, attached in -`backend-server.js`; never open-coded in the handler. +New param shape → new guard there, attached in `backend-server.js`. ### Response enrichment lives in middleware too @@ -121,11 +125,6 @@ tokens). `persona` strips the framing headers (`host` / `content-length` / …) first, as it re-serializes the body. Do **not** replicate for third-party upstreams; those get only what's explicitly needed. -### Defensive method gates - -Some handlers keep a `req.method !== 'GET'` branch although the route already -gates the method — smoke tests assert on it. Leave it when a test covers it. - ## Edge caching Every `/api/*` response defaults to `Cache-Control: no-store`; slowly-changing @@ -143,7 +142,8 @@ their caching belongs to the upstream owning the auth context. - Handlers get smoke tests in `tests/api-handlers.test.js` (method gating, param branches, "API key missing" early returns); a new or touched handler ships its - block in the same change. + block in the same change. A `req.method !== 'GET'` branch the route already + gates stays when a test asserts on it. - Never hit real upstreams: assert before the first `fetchUpstream`, or stub `globalThis.fetch` (restored in the shared `afterEach`). - Middleware is covered by `tests/guards.test.js` (don't duplicate per handler); diff --git a/api/asn-profile.js b/api/asn-profile.js index 1fc54b41..6a8231f7 100644 --- a/api/asn-profile.js +++ b/api/asn-profile.js @@ -1,16 +1,20 @@ // /api/asn-profile?asn= — everything the ASN Profile page shows, in one // answer: Radar ASN summary + prefix list (common/cf-radar.js), CAIDA // topology and neighbours (./asn-connectivity.js), RDAP autnum -// (common/rdap.js), ASRank (common/asrank.js) and, when the private API is -// configured, reputation (common/asn-reputation.js). Composition, deadlines and -// statuses live in common/asn-profile.js. `asn` arrives numeric +// (common/rdap.js), ASRank (common/asrank.js), the local PeeringDB index +// (common/peeringdb-db.js) and, when the private API is configured, +// reputation (common/asn-reputation.js). Composition, deadlines and statuses +// live in common/asn-profile.js. `asn` arrives numeric // (requireValidASN). 200 unless every configured source failed (502). import { RADAR_VIEWS, hasRadarApiKey } from '../common/cf-radar.js'; import { rdapAutnum, isAutnumMissing } from '../common/rdap.js'; import { queryAsRank } from '../common/asrank.js'; import { requestAsnReputation } from '../common/asn-reputation.js'; -import { buildSectionLoaders, composeAsnProfile, allSourcesFailed } from '../common/asn-profile.js'; +import { isPeeringdbLoaded, lookupPeeringdb } from '../common/peeringdb-db.js'; +import { + SOURCE_TIMEOUTS, buildSectionLoaders, composeAsnProfile, allSourcesFailed, +} from '../common/asn-profile.js'; import { getAsnConnectivity } from './asn-connectivity.js'; export default async (req, res) => { @@ -20,11 +24,13 @@ export default async (req, res) => { fetchRadarAsn: (n) => RADAR_VIEWS.asn.fetch({ asn: String(n) }), fetchRadarPrefixes: (n) => RADAR_VIEWS['bgp-prefixes'].fetch({ asn: String(n) }), getConnectivity: getAsnConnectivity, - rdapAutnum, + rdapAutnum: (n) => rdapAutnum(n, { timeoutMs: SOURCE_TIMEOUTS.autnum }), isAutnumMissing, - queryAsRank, + queryAsRank: (n) => queryAsRank(n, { timeoutMs: SOURCE_TIMEOUTS.rank }), // Private-API pass-through: the caller's headers go upstream. - requestReputation: (n) => requestAsnReputation(n, req.headers), + requestReputation: (n) => requestAsnReputation(n, req.headers, { timeoutMs: SOURCE_TIMEOUTS.reputation }), + isPeeringdbLoaded, + lookupPeeringdb, }); const body = await composeAsnProfile(asn, loaders); if (allSourcesFailed(body)) { diff --git a/backend-server.js b/backend-server.js index ade2a6ac..577a4a85 100644 --- a/backend-server.js +++ b/backend-server.js @@ -336,10 +336,11 @@ if (process.env.SENTRY_DSN_BACKEND) { } -// Bootstrap every offline dataset (MaxMind, CAIDA) before accepting traffic -// so we never serve mid-download. Each step is non-fatal: a failure leaves -// the dependent API in a degraded state (MaxMind → 503; CAIDA → empty graph -// or RIPEstat fallback) but doesn't block the listener. +// Bootstrap every offline dataset (MaxMind, CAIDA incl. the PeeringDB +// mirror) before accepting traffic so we never serve mid-download. Each step +// is non-fatal: a failure leaves the dependent API in a degraded state +// (MaxMind → 503; CAIDA → empty graph, RIPEstat fallback or no peering +// section) but doesn't block the listener. async function bootBackend() { await bootstrapMaxMindIfMissing({ reload: reloadMaxMindDatabases }); await reloadMaxMindDatabases('startup').catch(() => { diff --git a/common/asn-profile.js b/common/asn-profile.js index c3d87481..dc877fc6 100644 --- a/common/asn-profile.js +++ b/common/asn-profile.js @@ -5,26 +5,49 @@ // loaders, so tests need no network. // // Response shape (one key per section, plus `status`): -// { asn, status: { radar, prefixes, connectivity, whois, rank, reputation }, -// radar, prefixes, connectivity, whois, rank, reputation } +// { asn, status: { radar, prefixes, connectivity, whois, rank, reputation, peeringdb }, +// radar, prefixes, connectivity, whois, rank, reputation, peeringdb } // status values: 'ok' (data present) · 'empty' (source answered, nothing // for this AS) · 'error' (failed / timed out) · 'disabled' (not configured // on this deployment). Section data is non-null only for 'ok'. import logger from './logger.js'; -export const SECTIONS = ['radar', 'prefixes', 'connectivity', 'whois', 'rank', 'reputation']; +export const SECTIONS = ['radar', 'prefixes', 'connectivity', 'whois', 'rank', 'reputation', 'peeringdb']; -// Per-section budget (ms). Each sits just above the source's own upstream -// timeout where it has one (ASRank 4s, RDAP 5s), and cuts Radar's 8 s -// per-segment default short, so the whole answer is bounded by the largest. +// Upstream timeouts api/asn-profile.js hands to the sources that take one. +export const SOURCE_TIMEOUTS = { autnum: 5000, rank: 4000, reputation: 4000 }; + +const UPSTREAM_DEFAULT_MS = 8000; // fetchUpstream's default + +// Worst-case time (ms) each source's own upstream timeouts allow, so a +// source that degrades internally still answers before its deadline: +// radar 7 parallel Radar calls at the fetchUpstream default +// prefixes one Radar pfx2as call + local MaxMind +// whois IANA bootstrap (default, cached 24 h), then RDAP autnum +// connectivity local CAIDA + RIPEstat name fallback (3 s, in parallel) +// rank / reputation ASRank GraphQL / private API +// peeringdb local index, no I/O +export const INNER_TIMEOUTS = { + radar: UPSTREAM_DEFAULT_MS, + prefixes: UPSTREAM_DEFAULT_MS, + whois: UPSTREAM_DEFAULT_MS + SOURCE_TIMEOUTS.autnum, + connectivity: 3000, + rank: SOURCE_TIMEOUTS.rank, + reputation: SOURCE_TIMEOUTS.reputation, + peeringdb: 0, +}; + +// Per-section budget (ms): each sits above its source's worst case +// (INNER_TIMEOUTS), so the whole answer is bounded by the largest. export const DEADLINES = { - radar: 6500, - prefixes: 6500, + radar: 10000, + prefixes: 10000, connectivity: 5000, - whois: 6500, + whois: 14000, rank: 5000, reputation: 5000, + peeringdb: 1000, }; // Reject with a code 'deadline' error when `promise` outlives `ms`. The @@ -76,13 +99,21 @@ export const classifyReputation = (answer) => { return result(answer.data?.found === true ? 'ok' : 'empty', answer.data); }; +// `loaded` false = no PeeringDB index on this deployment; `record` is +// lookupPeeringdb's (null when the AS has none). +export const classifyPeeringdb = (loaded, record) => { + if (!loaded) return result('disabled'); + return result(record ? 'ok' : 'empty', record); +}; + // -- loaders ------------------------------------------------------------------ // Section loaders over the real data functions (injected — see // api/asn-profile.js). Each resolves { status, data } or throws. // deps: { hasRadarKey(), fetchRadarAsn(asn), fetchRadarPrefixes(asn), // getConnectivity(asn), rdapAutnum(asn), isAutnumMissing(err), -// queryAsRank(asn), requestReputation(asn) } +// queryAsRank(asn), requestReputation(asn), +// isPeeringdbLoaded(), lookupPeeringdb(asn) } export const buildSectionLoaders = (deps) => ({ radar: async (asn) => (deps.hasRadarKey() ? classifyRadar(await deps.fetchRadarAsn(asn)) : result('disabled')), prefixes: async (asn) => (deps.hasRadarKey() ? classifyPrefixes(await deps.fetchRadarPrefixes(asn)) : result('disabled')), @@ -97,6 +128,7 @@ export const buildSectionLoaders = (deps) => ({ }, rank: async (asn) => classifyRank(await deps.queryAsRank(asn)), reputation: async (asn) => classifyReputation(await deps.requestReputation(asn)), + peeringdb: async (asn) => classifyPeeringdb(deps.isPeeringdbLoaded(), deps.lookupPeeringdb(asn)), }); // Run every loader under its deadline and assemble the response body. diff --git a/common/caida-updater.js b/common/caida-updater.js index 0a7ab641..41229158 100644 --- a/common/caida-updater.js +++ b/common/caida-updater.js @@ -4,11 +4,16 @@ // pattern. // // Registered datasets: -// - as2org (AS → org-name mapping) common/as-org-db -// - as-rel (AS relationships, p2c) common/as-rel-db +// - as2org (AS → org-name mapping) common/as-org-db +// - as-rel (AS relationships, p2c) common/as-rel-db +// - peeringdb (PeeringDB dump → per-ASN index) common/peeringdb-db // -// Both share CAIDA_AUTO_UPDATE=true to gate the periodic scheduler -// (off by default); bootstrap always runs so a fresh checkout works. +// All share CAIDA_AUTO_UPDATE=true to gate the periodic scheduler (off by +// default); bootstrap always runs so a fresh checkout works. A row may add +// `enabled()` (skipped by both when false — peeringdb needs the Cloudflare +// key, as the ASN Profile is hidden without it), leave out `decompress` +// (the download is used as is) and add `distill(rawPath, tempDir, { signal })` +// (turns the download into the published file; the raw file is deleted). import fs from 'fs'; import fsp from 'fs/promises'; @@ -21,6 +26,9 @@ import { withCronMonitor } from './sentry-cron.js'; import { createDecompressor } from './decompress.js'; import { AS_ORG_DB_DIR, AS_ORG_FILE, reloadAsOrgDatabase } from './as-org-db.js'; import { AS_REL_DB_DIR, AS_REL_FILE, reloadAsRelDatabase } from './as-rel-db.js'; +import { PEERINGDB_DB_DIR, PEERINGDB_FILE, readPeeringdbIndex, reloadPeeringdbDatabase } from './peeringdb-db.js'; +import { distillPeeringdbDump } from './peeringdb-distill.js'; +import { hasRadarApiKey } from './cf-radar.js'; const UPDATE_INTERVAL_MS = 24 * 60 * 60 * 1000; const INITIAL_UPDATE_DELAY_MS = 60 * 1000; @@ -29,7 +37,8 @@ const BOOTSTRAP_TIMEOUT_MS = 2 * 60 * 1000; const STATE_FILE = '.caida-update-state.json'; const LOCK_FILE = '.caida-update.lock'; -const datasets = [ +// Exported for tests (rows are re-pointed at a temp dir there). +export const datasets = [ { id: 'as2org', dbDir: AS_ORG_DB_DIR, @@ -68,6 +77,23 @@ const datasets = [ validate: validateAsRel, reload: reloadAsRelDatabase, }, + { + id: 'peeringdb', + dbDir: PEERINGDB_DB_DIR, + canonicalFile: PEERINGDB_FILE, + archiveExt: '.json', + enabled: hasRadarApiKey, + findRemote: findPeeringdbDump, + // ~116 MB raw dump → ~6 MB index; only the index is published. + distill: async (rawPath, tempDir, { signal } = {}) => { + const outPath = path.join(tempDir, PEERINGDB_FILE); + const counts = await distillPeeringdbDump(rawPath, outPath, { signal }); + logger.info({ dataset: 'peeringdb', ...counts }, 'PeeringDB dump distilled'); + return outPath; + }, + validate: validatePeeringdb, + reload: reloadPeeringdbDatabase, + }, ]; let schedulerStarted = false; @@ -82,7 +108,7 @@ const updateInProgress = new Set(); * Never throws. */ export async function bootstrapCaidaIfMissing() { - for (const dataset of datasets) { + for (const dataset of datasets.filter(isDatasetEnabled)) { try { await bootstrapDataset(dataset); } catch (error) { @@ -113,13 +139,13 @@ export function startCaidaAutoUpdate() { setInterval(run, UPDATE_INTERVAL_MS).unref?.(); const nextRunAt = new Date(Date.now() + INITIAL_UPDATE_DELAY_MS); - logger.info(`🗓️ CAIDA auto update plan: next check at ${nextRunAt.toLocaleString('en-US', { hour12: false })}, then every 24 hours (${datasets.length} datasets)`); + logger.info(`🗓️ CAIDA auto update plan: next check at ${nextRunAt.toLocaleString('en-US', { hour12: false })}, then every 24 hours (${datasets.filter(isDatasetEnabled).length} datasets)`); } // ---------- Orchestration ---------- async function runAllUpdates() { - for (const dataset of datasets) { + for (const dataset of datasets.filter(isDatasetEnabled)) { try { await updateDataset(dataset); } catch (error) { @@ -128,7 +154,8 @@ async function runAllUpdates() { } } -async function bootstrapDataset(dataset) { +// Exported for tests (with a temp-dir dataset row); not part of the API. +export async function bootstrapDataset(dataset) { await fsp.mkdir(dataset.dbDir, { recursive: true }); await clearOrphanedLock(dataset); @@ -163,7 +190,8 @@ async function bootstrapDataset(dataset) { } } -async function updateDataset(dataset, { signal, reloadReason = 'auto update' } = {}) { +// Exported for tests, like bootstrapDataset. +export async function updateDataset(dataset, { signal, reloadReason = 'auto update' } = {}) { if (updateInProgress.has(dataset.id)) { return { updated: false, reason: 'already-running' }; } @@ -201,7 +229,17 @@ async function downloadAndPublish(dataset, tempDir, { signal } = {}) { return { updated: false, reason: 'not-modified' }; } - const stagedPath = await downloadAndDecompress(dataset, remote.url, tempDir, { signal }); + let stagedPath = await downloadAndDecompress(dataset, remote.url, tempDir, { signal }); + if (dataset.distill) { + const rawPath = stagedPath; + try { + stagedPath = await dataset.distill(rawPath, tempDir, { signal }); + } catch (error) { + throw new Error(`${dataset.id} distill phase failed: ${error.message}`); + } finally { + await fsp.rm(rawPath, { force: true }); + } + } await dataset.validate(stagedPath); await publishFile(stagedPath, targetPath); @@ -214,8 +252,9 @@ async function downloadAndPublish(dataset, tempDir, { signal } = {}) { return { updated: true, identifier: remote.identifier }; } -// Stream archive to disk, then decompress to staged .txt. Split so errors -// clearly identify which phase failed (network vs decompression). +// Stream archive to disk, then decompress to staged .txt (a row without +// `decompress` stages the download itself). Split so errors clearly +// identify which phase failed (network vs decompression). async function downloadAndDecompress(dataset, url, tempDir, { signal } = {}) { const archivePath = path.join(tempDir, `archive${dataset.archiveExt}`); const stagedPath = path.join(tempDir, dataset.canonicalFile); @@ -229,6 +268,7 @@ async function downloadAndDecompress(dataset, url, tempDir, { signal } = {}) { } catch (error) { throw new Error(`${dataset.id} download phase failed: ${error.message}`); } + if (!dataset.decompress) return archivePath; try { await pipeline( @@ -284,6 +324,37 @@ async function validateAsRel(stagedPath) { } } +// Healthy index has ~33k networks; <20k indicates truncation or schema change. +async function validatePeeringdb(stagedPath) { + const MIN_VALID = 20000; + const { nets } = readPeeringdbIndex(stagedPath); + if (nets.size < MIN_VALID) { + throw new Error(`Staged PeeringDB index has only ${nets.size} networks; refusing to publish`); + } +} + +// ---------- Remote lookups ---------- + +// CAIDA mirrors the full PeeringDB dump daily (a day late) under YYYY/MM/. +// List the current UTC month, then the previous one for the days before the +// month's first dump lands; the newest filename wins. Exported for tests. +export async function findPeeringdbDump({ signal, now = new Date() } = {}) { + const base = 'https://publicdata.caida.org/datasets/peeringdb/'; + for (const back of [0, 1]) { + const month = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - back, 1)); + const dir = `${base}${month.getUTCFullYear()}/${String(month.getUTCMonth() + 1).padStart(2, '0')}/`; + const res = await fetch(dir, { signal }); + if (res.status === 404) continue; + if (!res.ok) throw new Error(`Directory listing failed: HTTP ${res.status}`); + const html = await res.text(); + const names = [...html.matchAll(/\bpeeringdb_2_dump_\d{4}_\d{2}_\d{2}\.json\b/g)].map(m => m[0]); + if (names.length === 0) continue; + names.sort((a, b) => b.localeCompare(a)); + return { url: dir + names[0], identifier: names[0] }; + } + throw new Error('No PeeringDB dump in the current or previous month listing'); +} + // ---------- Generic helpers ---------- async function publishFile(stagedPath, targetPath) { @@ -306,13 +377,21 @@ async function publishFile(stagedPath, targetPath) { } } +// A row without `enabled` is always on. Exported for tests. +export function isDatasetEnabled(dataset) { + return !dataset.enabled || Boolean(dataset.enabled()); +} + function isAutoUpdateEnabled() { return process.env.CAIDA_AUTO_UPDATE === 'true'; } +// Any non-dot file with the canonical file's extension (.txt for the +// CAIDA rows, so manual downloads under another name still count). function snapshotExists(dataset) { if (!fs.existsSync(dataset.dbDir)) return false; - return fs.readdirSync(dataset.dbDir).some(f => f.endsWith('.txt')); + const ext = path.extname(dataset.canonicalFile); + return fs.readdirSync(dataset.dbDir).some(f => f.endsWith(ext) && !f.startsWith('.')); } // At boot, any pre-existing lock is necessarily from a previous crashed run diff --git a/common/peeringdb-db.js b/common/peeringdb-db.js new file mode 100644 index 00000000..db7f9d18 --- /dev/null +++ b/common/peeringdb-db.js @@ -0,0 +1,247 @@ +// Local PeeringDB lookup: a network's self-reported peering profile (types, +// website, scope, traffic, ratio, policy, IRR as-set), the exchanges it is +// on and the facilities it is in. common/caida-updater.js downloads CAIDA's +// daily mirror of the full PeeringDB dump and distills it +// (common/peeringdb-distill.js → buildPeeringdbIndex below); only the +// compact per-ASN index ever lands in this directory. Contact data (poc, +// org addresses) is never read. +// +// Index file (JSON): +// { v: 1, +// ix: { : [name, city, country] }, +// fac: { : [name, city, country] }, +// nets: { : { website, types[], scope, traffic, ratio, policy, irr, +// ix: [[ix id, Mbps summed over ports, rs peer 1|0]], +// fac: [fac id] } } } +// Empty / "Not Disclosed" fields are omitted; a network with nothing left +// is dropped. IX and facility rows are stored once and referenced by id. +// +// Sources: https://publicdata.caida.org/datasets/peeringdb/ (CAIDA AUA) · +// https://www.peeringdb.com + +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import logger from './logger.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +export const PEERINGDB_DB_DIR = path.join(__dirname, 'peeringdb-db'); +export const PEERINGDB_FILE = 'peeringdb-index.json'; +export const PEERINGDB_INDEX_VERSION = 1; + +// The four dump tables the index is built from. +export const PEERINGDB_TABLES = ['net', 'ix', 'netixlan', 'netfac']; + +/* ------------------------------------------------------------------ */ +/* Distill: dump rows → index */ +/* ------------------------------------------------------------------ */ + +const UNDISCLOSED = 'not disclosed'; + +// Trimmed text, or undefined for empty / "Not Disclosed". +export const cleanText = (value) => { + if (typeof value !== 'string') return undefined; + const text = value.trim(); + return text && text.toLowerCase() !== UNDISCLOSED ? text : undefined; +}; + +// http(s) URLs only, and none with userinfo — "name@host" is how an +// e-mail address typed into the field would otherwise pass as a link. +const cleanWebsite = (value) => { + const text = cleanText(value); + if (!text) return undefined; + try { + const url = new URL(text); + const web = url.protocol === 'http:' || url.protocol === 'https:'; + return web && !url.username && !url.password ? text : undefined; + } catch { + return undefined; + } +}; + +const isLive = (row) => !row.status || row.status === 'ok'; + +// Per-table projection to the fields the index needs, applied to each row +// as it is scanned so nothing else is kept. +export const projectRow = { + net: (r) => ({ + id: r.id, + asn: Number(r.asn), + website: cleanWebsite(r.website), + types: Array.isArray(r.info_types) ? r.info_types.map(cleanText).filter(Boolean) : [], + scope: cleanText(r.info_scope), + traffic: cleanText(r.info_traffic), + ratio: cleanText(r.info_ratio), + policy: cleanText(r.policy_general), + irr: cleanText(r.irr_as_set), + live: isLive(r), + }), + ix: (r) => ({ id: r.id, name: cleanText(r.name), city: cleanText(r.city), country: cleanText(r.country), live: isLive(r) }), + netixlan: (r) => ({ + asn: Number(r.asn), + ixId: r.ix_id, + speed: Number(r.speed) || 0, + rs: r.is_rs_peer === true, + live: isLive(r) && r.operational !== false, + }), + netfac: (r) => ({ + netId: r.net_id, + facId: r.fac_id, + name: cleanText(r.name), + city: cleanText(r.city), + country: cleanText(r.country), + live: isLive(r), + }), +}; + +const place = (row) => [row.name, row.city || '', row.country || '']; + +/** + * Projected rows of the four tables → index object (shape in the header). + * Pure; the streaming distiller and the tests share it. + */ +export const buildPeeringdbIndex = ({ net = [], ix = [], netixlan = [], netfac = [] }) => { + const ixById = new Map(ix.filter((r) => r.live && r.name).map((r) => [r.id, r])); + const asnByNetId = new Map(net.map((r) => [r.id, r.asn])); + const ixTable = {}; + const facTable = {}; + + // asn → Map; several ports at one IX sum. + const portsByAsn = new Map(); + for (const row of netixlan) { + if (!row.live || !row.asn) continue; + const exchange = ixById.get(row.ixId); + if (!exchange) continue; + let atIx = portsByAsn.get(row.asn); + if (!atIx) portsByAsn.set(row.asn, (atIx = new Map())); + const entry = atIx.get(exchange.id); + if (entry) { + entry[1] += row.speed; + entry[2] = entry[2] || (row.rs ? 1 : 0); + } else { + atIx.set(exchange.id, [exchange.id, row.speed, row.rs ? 1 : 0]); + ixTable[exchange.id] ??= place(exchange); + } + } + + // netfac carries the facility's own name / city / country. + const facsByAsn = new Map(); + for (const row of netfac) { + if (!row.live || !row.name) continue; + const asn = asnByNetId.get(row.netId); + if (!asn) continue; + let ids = facsByAsn.get(asn); + if (!ids) facsByAsn.set(asn, (ids = new Set())); + ids.add(row.facId); + facTable[row.facId] ??= place(row); + } + + const byPlace = (a, b) => a[2].localeCompare(b[2]) || a[1].localeCompare(b[1]) || a[0].localeCompare(b[0]); + const nets = {}; + for (const row of net) { + if (!row.live || !row.asn || nets[row.asn]) continue; + const entry = {}; + for (const key of ['website', 'scope', 'traffic', 'ratio', 'policy', 'irr']) { + if (row[key]) entry[key] = row[key]; + } + if (row.types.length) entry.types = row.types; + const ports = [...(portsByAsn.get(row.asn)?.values() || [])] + .sort((a, b) => b[1] - a[1] || ixTable[a[0]][0].localeCompare(ixTable[b[0]][0])); + if (ports.length) entry.ix = ports; + const facs = [...(facsByAsn.get(row.asn) || [])].sort((a, b) => byPlace(facTable[a], facTable[b])); + if (facs.length) entry.fac = facs; + if (Object.keys(entry).length) nets[row.asn] = entry; + } + + // Keep only the IX / facility rows some network still references. + const used = (table, key) => { + const ids = new Set(); + for (const entry of Object.values(nets)) { + for (const item of entry[key] || []) ids.add(key === 'ix' ? item[0] : item); + } + return Object.fromEntries(Object.entries(table).filter(([id]) => ids.has(Number(id)))); + }; + return { v: PEERINGDB_INDEX_VERSION, ix: used(ixTable, 'ix'), fac: used(facTable, 'fac'), nets }; +}; + +/* ------------------------------------------------------------------ */ +/* Load + lookup */ +/* ------------------------------------------------------------------ */ + +const toPlaceMap = (table) => new Map(Object.entries(table || {}).map(([id, [name, city, country]]) => [ + Number(id), { name, city: city || null, country: country || null }, +])); + +/** + * Read an index file into lookup tables { nets, ix, fac } (Maps keyed by + * number). The file text and the parsed object stay local, so only the + * Maps outlive the call. Throws on a malformed or wrong-version file. + */ +export const readPeeringdbIndex = (filePath) => { + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); + if (parsed?.v !== PEERINGDB_INDEX_VERSION || !parsed.nets) { + throw new Error(`unsupported PeeringDB index (v=${parsed?.v})`); + } + const nets = new Map(); + for (const asn of Object.keys(parsed.nets)) nets.set(Number(asn), parsed.nets[asn]); + return { nets, ix: toPlaceMap(parsed.ix), fac: toPlaceMap(parsed.fac) }; +}; + +/** One network's record from loaded tables, expanded for the API; null when absent. */ +export const expandNet = (tables, asn) => { + const entry = tables?.nets.get(Number(asn)); + if (!entry) return null; + return { + website: entry.website || null, + types: entry.types || [], + scope: entry.scope || null, + traffic: entry.traffic || null, + ratio: entry.ratio || null, + policy: entry.policy || null, + irrAsSet: entry.irr || null, + ixs: (entry.ix || []).flatMap(([id, speed, rs]) => { + const exchange = tables.ix.get(id); + return exchange ? [{ ...exchange, speed, rsPeer: rs === 1 }] : []; + }), + facilities: (entry.fac || []).flatMap((id) => { + const facility = tables.fac.get(id); + return facility ? [{ ...facility }] : []; + }), + }; +}; + +let tables = null; +let loadedFrom = null; + +const loadDatabase = () => { + const filePath = path.join(PEERINGDB_DB_DIR, PEERINGDB_FILE); + if (!fs.existsSync(filePath)) { + // Info, not warn: deployments without a Cloudflare key never fetch one. + logger.info({ dir: PEERINGDB_DB_DIR }, '⚠️ PeeringDB index not present; ASN Profile shows no peering data'); + tables = null; + loadedFrom = null; + return; + } + const start = Date.now(); + try { + tables = readPeeringdbIndex(filePath); + loadedFrom = PEERINGDB_FILE; + logger.info(`📦 PeeringDB index loaded — ${tables.nets.size} networks, ${tables.ix.size} exchanges, ${tables.fac.size} facilities in ${Date.now() - start}ms`); + } catch (error) { + logger.warn({ err: error, path: filePath }, '⚠️ Failed to parse PeeringDB index'); + } +}; + +loadDatabase(); + +/** Reload the index after caida-updater publishes a fresh one. */ +export const reloadPeeringdbDatabase = (reason = 'reload') => { + logger.info(`🔄 Reloading PeeringDB index (${reason})`); + loadDatabase(); +}; + +/** Whether an index is loaded (false until the updater has built one). */ +export const isPeeringdbLoaded = () => loadedFrom !== null; + +/** The network's PeeringDB record (see expandNet), or null. */ +export const lookupPeeringdb = (asn) => expandNet(tables, asn); diff --git a/common/peeringdb-distill.js b/common/peeringdb-distill.js new file mode 100644 index 00000000..87d47033 --- /dev/null +++ b/common/peeringdb-distill.js @@ -0,0 +1,174 @@ +// Streaming distiller: full PeeringDB dump (CAIDA mirror, ~116 MB JSON) → +// the compact per-ASN index common/peeringdb-db.js serves. Built for small +// instances: the dump is never parsed whole. A byte-level scanner walks the +// dump's fixed envelope +// { "": { "data": [ {row}, {row}, … ], "meta": {…} }, … } +// and JSON.parses one row at a time, only inside the wanted tables' "data" +// arrays, projecting it to the needed fields at once; every other table +// (poc, org, fac, …) is skipped byte by byte and never materialized. The +// index is written in batches rather than as one string. +// +// Scanning raw bytes is UTF-8 safe: continuation bytes (0x80–0xBF) never +// equal '"', '\', ':', ',' or a bracket. Reads are async, one reused 1 MB +// buffer at a time, so a running server keeps serving between chunks. + +import fsp from 'fs/promises'; +import { PEERINGDB_TABLES, projectRow, buildPeeringdbIndex } from './peeringdb-db.js'; + +const QUOTE = 0x22; +const BACKSLASH = 0x5c; +const COLON = 0x3a; +const COMMA = 0x2c; +const OPEN_OBJECT = 0x7b; +const CLOSE_OBJECT = 0x7d; +const OPEN_ARRAY = 0x5b; +const CLOSE_ARRAY = 0x5d; + +const CHUNK_BYTES = 1 << 20; +const WRITE_BATCH_CHARS = 1 << 20; + +/** + * Stream `filePath` and call `onRow(table, row)` for every row object in + * the "data" array of each table named in `wanted`. Resolves the number of + * rows seen; throws on a truncated document or when `signal` aborts. + */ +export const scanDumpRows = async (filePath, wanted, onRow, { signal } = {}) => { + let depth = 0; + let inString = false; + let escaped = false; + // Container kind per depth ('o' / 'a') and whether the next string at + // depth 1 or 2 is a key; only those two levels name tables / fields. + const kinds = []; + let expectKey = false; + let keyParts = null; + let keyStart = -1; + let topKey = null; + let tableKey = null; + let rowParts = null; + let rowStart = -1; + let rows = 0; + let sawDocument = false; + + const buf = Buffer.allocUnsafe(CHUNK_BYTES); + const handle = await fsp.open(filePath, 'r'); + try { + for (;;) { + signal?.throwIfAborted(); + const { bytesRead } = await handle.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + const chunk = buf.subarray(0, bytesRead); + for (let i = 0; i < chunk.length; i++) { + const b = chunk[i]; + if (inString) { + if (escaped) escaped = false; + else if (b === BACKSLASH) escaped = true; + else if (b === QUOTE) { + inString = false; + if (keyParts) { + keyParts.push(chunk.subarray(keyStart, i)); + const key = Buffer.concat(keyParts).toString('utf8'); + if (depth === 1) { + topKey = key; + tableKey = null; + } else { + tableKey = key; + } + keyParts = null; + } + } + continue; + } + if (b === QUOTE) { + inString = true; + if ((depth === 1 || depth === 2) && kinds[depth] === 'o' && expectKey) { + keyParts = []; + keyStart = i + 1; + } + } else if (b === COLON) { + if (depth <= 2) expectKey = false; + } else if (b === COMMA) { + if (depth <= 2) expectKey = kinds[depth] === 'o'; + } else if (b === OPEN_OBJECT || b === OPEN_ARRAY) { + depth++; + sawDocument = true; + if (depth <= 3) kinds[depth] = b === OPEN_OBJECT ? 'o' : 'a'; + if (depth <= 2) expectKey = b === OPEN_OBJECT; + if (depth === 4 && b === OPEN_OBJECT && kinds[3] === 'a' + && tableKey === 'data' && wanted.has(topKey)) { + rowParts = []; + rowStart = i; + } + } else if (b === CLOSE_OBJECT || b === CLOSE_ARRAY) { + if (depth === 4 && rowParts) { + const text = rowParts.length + ? Buffer.concat([...rowParts, chunk.subarray(rowStart, i + 1)]).toString('utf8') + : chunk.toString('utf8', rowStart, i + 1); + onRow(topKey, JSON.parse(text)); + rows++; + rowParts = null; + } + depth--; + } + } + // A row or key straddling the boundary outlives the reused buffer. + if (rowParts) { + rowParts.push(Buffer.from(chunk.subarray(rowStart))); + rowStart = 0; + } + if (keyParts) { + keyParts.push(Buffer.from(chunk.subarray(keyStart))); + keyStart = 0; + } + } + } finally { + await handle.close(); + } + if (!sawDocument || depth !== 0 || inString) { + throw new Error(`truncated PeeringDB dump (depth ${depth})`); + } + return rows; +}; + +/** Write an index object as JSON in ~1 MB batches; same bytes as JSON.stringify. */ +export const writeIndexFile = async (outPath, index) => { + const handle = await fsp.open(outPath, 'w'); + try { + let pending = `{"v":${JSON.stringify(index.v)},"ix":${JSON.stringify(index.ix)},"fac":${JSON.stringify(index.fac)},"nets":{`; + let first = true; + for (const asn of Object.keys(index.nets)) { + pending += `${first ? '' : ','}${JSON.stringify(asn)}:${JSON.stringify(index.nets[asn])}`; + first = false; + if (pending.length >= WRITE_BATCH_CHARS) { + await handle.write(pending); + pending = ''; + } + } + await handle.write(`${pending}}}`); + } finally { + await handle.close(); + } +}; + +/** + * Dump file → index file. Resolves counts for the log; throws when the dump + * lacks one of the four tables (a schema change must not publish). + */ +export const distillPeeringdbDump = async (dumpPath, outPath, { signal } = {}) => { + const wanted = new Set(PEERINGDB_TABLES); + const tables = Object.fromEntries(PEERINGDB_TABLES.map((name) => [name, []])); + const rows = await scanDumpRows(dumpPath, wanted, (table, row) => { + tables[table].push(projectRow[table](row)); + }, { signal }); + for (const name of PEERINGDB_TABLES) { + if (!tables[name].length) throw new Error(`PeeringDB dump has no ${name} rows`); + } + const index = buildPeeringdbIndex(tables); + for (const name of PEERINGDB_TABLES) tables[name] = null; // garbage before the write + await writeIndexFile(outPath, index); + return { + rows, + networks: Object.keys(index.nets).length, + exchanges: Object.keys(index.ix).length, + facilities: Object.keys(index.fac).length, + }; +}; diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index 3064190d..a9258704 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -68,6 +68,9 @@ grid and the nav; deep links (`?tool=`, `/tools/:slug`) are never gated. section (reputation included — no `configs.originalSite` check) renders only when its `status` is `ok`. `ASNConnectivity.vue` gets `:expandable="false" :bordered="false"`; its defaults keep IPCard / QueryIP unchanged. +- `peeringdb` is operator self-reporting: vocabularies → locale keys + (`PEERING_ENUMS`), names / cities as written. `ASN_PROFILE_TIMEOUT_MS` stays + above the largest backend section deadline (tested). - ASN Info's footer links `/tools/asn?q=AS` (hidden when gated off); in QueryIP it emits `open-tool` via IpDetailPanel so the dialog closes first. @@ -111,9 +114,7 @@ stateless panel, `success` protective state on, `secondary` dock; ≤ 2 accents. Every "business state → color" mapping goes through `use-status-tone.js` (`wait` / `ok-fast` / `ok-slow` / `fail`), normally via `ipFieldTone()`. -### Canonical patterns - -Copy the named exemplar instead of re-inventing: +### Canonical patterns — copy the named exemplar, don't re-invent - **Trigger button** — `variant="action"` + `` + `:disabled` (QueryIP, Whois). - **Input + icon trigger** — flex row, compact icon Button, no text label (QueryIP). @@ -145,6 +146,5 @@ Copy the named exemplar instead of re-inventing: ## Testing -Composables and utils are the target (`tests/composable-*.test.js`); Vue rendering -and browser APIs are out of scope. Visual changes can't be self-tested — say so -and let the user verify in `pnpm dev`. +Composables and utils are the target (`tests/composable-*.test.js`); rendering / +browser APIs are out of scope — visual changes need the user in `pnpm dev`. diff --git a/frontend/components/advanced-tools/AsnProfile.vue b/frontend/components/advanced-tools/AsnProfile.vue index 5972b452..d9e7b967 100644 --- a/frontend/components/advanced-tools/AsnProfile.vue +++ b/frontend/components/advanced-tools/AsnProfile.vue @@ -5,7 +5,8 @@ failed is simply left out. Order: hero (identity, key facts, registration) beside a compact reputation card when there is one → traffic & quality beside top - countries → prefixes → connectivity (+ customer cone, topology). + countries → prefixes → connectivity (+ customer cone, topology) → + peering & facilities (PeeringDB, self-reported). One source per concept: relationships = local CAIDA (graph, counts and lists alike); prefixes, addresses and RPKI = the pfx2as list; identity = Radar → RDAP → ASRank. The query rides the URL as `?q=` (shareable, @@ -73,8 +74,21 @@ -
- {{ t('asnprofile.tier1') }} + +
+
+ {{ t('asnprofile.tier1') }} + + {{ enumLabel('types', type) }} +
+ + {{ enumLabel('scopes', peerHero.scope) }} + + {{ peerHero.website.label }}
@@ -334,8 +348,68 @@

{{ t('asnprofile.connectivity.none') }}

+ - + +
+
+

{{ t('asnprofile.peering.title') }}

+

{{ t('asnprofile.peering.desc') }}

+
+ + +
+
+
{{ t(`asnprofile.peering.${fact.key}`) }}
+
+ {{ fact.label }}
+
+
+ + +
+ +
+

+ {{ t('asnprofile.peering.exchanges', { count: nf.format(exchanges.length) }) }}

+
    +
  • + + +
    +
    {{ row.name }}
    +
    {{ row.city }}
    +
    + {{ t('asnprofile.peering.rs') }} + {{ formatPortSpeed(row.speed, nf.format) }} +
    • +
+ +
+ + +
+

+ {{ t('asnprofile.peering.facilities', { count: nf.format(facilities.length) }) }}

+
    +
  • + + +
    +
    {{ row.name }}
    +
    {{ row.city }}
    +
    +
    • +
+ +
+
@@ -351,18 +425,19 @@ import { computed, defineAsyncComponent, onMounted, ref, shallowRef, watch } fro import { useRoute, useRouter } from 'vue-router'; import { useI18n } from 'vue-i18n'; import { Icon } from '@iconify/vue'; -import { ChevronRight, Download, Search, ShieldAlert, ShieldCheck, ShieldX } from '@lucide/vue'; +import { ChevronRight, Download, ExternalLink, Globe, Search, ShieldAlert, ShieldCheck, ShieldX } from '@lucide/vue'; import { trackEvent } from '@/utils/analytics'; import { useStatusTone } from '@/composables/use-status-tone.js'; import { fetchWithTimeout } from '@/utils/fetch-with-timeout.js'; import { parseAsnInput } from '@/utils/asn-input.js'; import { buildTrafficPairs, pickConnectionQuality } from '@/utils/asn-metrics.js'; import { - asnProfileUrl, sectionData, isProfileEmpty, failedSources, + asnProfileUrl, ASN_PROFILE_TIMEOUT_MS, sectionData, isProfileEmpty, failedSources, hasTopology, isTier1Origin, heroIdentity, registrationFields, keyFacts, coneAsns, RPKI_STATES, shapePrefixRows, familyCounts, filterPrefixRows, rpkiSummary, prefixListText, countryShareRows, neighbourGroups, BASELINE_POSITION, reputationMeters, proxyListedCount, verdictLevel, VERDICT_TONE, + peeringEnumKey, peeringHero, peeringFacts, formatPortSpeed, exchangeRows, facilityRows, hasPeeringCard, visibleRows, } from '@/utils/asn-profile-view.js'; import { formatIsoDate } from '@/utils/time-utils.js'; import getCountryName from '@/data/country-name.js'; @@ -386,8 +461,6 @@ const route = useRoute(); const router = useRouter(); const { dotClass, textClass } = useStatusTone(); -// Well above the backend's worst case (its largest per-source deadline). -const REQUEST_TIMEOUT_MS = 15000; const tagClass = 'group h-7 rounded-full px-2.5 text-xs cursor-pointer'; const nf = computed(() => new Intl.NumberFormat(locale.value, { maximumFractionDigits: 1 })); @@ -409,6 +482,8 @@ const FIRST_ROWS = 10; const PAGE = 50; const familyFilter = ref('all'); const limit = ref(FIRST_ROWS); +const showAllExchanges = ref(false); +const showAllFacilities = ref(false); const loading = computed(() => state.value === 'loading'); @@ -420,8 +495,10 @@ const run = async (asn) => { rawOpen.value = false; familyFilter.value = 'all'; limit.value = FIRST_ROWS; + showAllExchanges.value = false; + showAllFacilities.value = false; try { - const response = await fetchWithTimeout(asnProfileUrl(asn), { timeoutMs: REQUEST_TIMEOUT_MS }); + const response = await fetchWithTimeout(asnProfileUrl(asn), { timeoutMs: ASN_PROFILE_TIMEOUT_MS }); const body = response.ok ? await response.json() : null; if (id !== runId) return; profile.value = body?.status ? body : null; @@ -474,6 +551,7 @@ const graph = computed(() => sectionData(profile.value, 'connectivity')); const whois = computed(() => sectionData(profile.value, 'whois')); const rank = computed(() => sectionData(profile.value, 'rank')); const reputation = computed(() => sectionData(profile.value, 'reputation')); +const peeringdb = computed(() => sectionData(profile.value, 'peeringdb')); const empty = computed(() => isProfileEmpty(profile.value)); const failed = computed(() => failedSources(profile.value)); @@ -584,4 +662,21 @@ const groups = computed(() => neighbourGroups(graph.value?.neighbours, NAME_PREV const connectivityInfos = computed(() => (graph.value ? { [String(profileAsn.value)]: { graph: graph.value } } : {})); const cone = computed(() => coneAsns(rank.value)); + +// ---- peering & facilities (PeeringDB) -------------------------------------- + +// PeeringDB vocabulary → translated label; anything else as written. +const enumLabel = (group, value) => { + const key = peeringEnumKey(group, value); + return key ? t(key) : value; +}; + +const peerHero = computed(() => peeringHero(peeringdb.value)); +const peerFacts = computed(() => peeringFacts(peeringdb.value) + .map((fact) => ({ ...fact, label: fact.group ? enumLabel(fact.group, fact.value) : fact.value }))); +const exchanges = computed(() => exchangeRows(peeringdb.value)); +const facilities = computed(() => facilityRows(peeringdb.value)); +const peeringCard = computed(() => hasPeeringCard(peeringdb.value)); +const shownExchanges = computed(() => visibleRows(exchanges.value, showAllExchanges.value, FIRST_ROWS)); +const shownFacilities = computed(() => visibleRows(facilities.value, showAllFacilities.value, FIRST_ROWS)); diff --git a/frontend/data/changelog.json b/frontend/data/changelog.json index acf8fca5..b004bf37 100644 --- a/frontend/data/changelog.json +++ b/frontend/data/changelog.json @@ -2144,12 +2144,12 @@ { "type": "add", "change": { - "en": "New tool: ASN Profile — a full profile of any autonomous system, covering identity and registration, scale, traffic, where its addresses are used, announced prefixes with RPKI status, and connectivity", - "zh": "新增工具：ASN 档案——任意自治系统的完整档案，涵盖身份与注册信息、规模、流量、地址的使用地区、宣告的前缀及其 RPKI 状态，以及互联关系", - "zh-TW": "新增工具：ASN 檔案——任意自治系統的完整檔案，涵蓋身分與註冊資訊、規模、流量、位址的使用地區、宣告的前綴及其 RPKI 狀態，以及互連關係", - "fr": "Nouvel outil : Profil ASN — le profil complet de n'importe quel système autonome : identité et enregistrement, taille, trafic, pays où ses adresses sont utilisées, préfixes annoncés avec leur statut RPKI et interconnexions", - "ru": "Новый инструмент: профиль ASN — полный профиль любой автономной системы: владелец и регистрация, масштаб, трафик, где используются её адреса, анонсируемые префиксы со статусом RPKI и связность", - "pt-BR": "Nova ferramenta: Perfil de ASN — o perfil completo de qualquer sistema autônomo: identidade e registro, porte, tráfego, onde seus endereços são usados, prefixos anunciados com status RPKI e conectividade" + "en": "New tool: ASN Profile — a full profile of any autonomous system, covering identity and registration, scale, traffic, where its addresses are used, announced prefixes with RPKI status, connectivity, and its peering policy, internet exchanges and data centres", + "zh": "新增工具：ASN 档案——任意自治系统的完整档案，涵盖身份与注册信息、规模、流量、地址的使用地区、宣告的前缀及其 RPKI 状态、互联关系，以及对等策略、接入的互联网交换中心与机房驻点", + "zh-TW": "新增工具：ASN 檔案——任意自治系統的完整檔案，涵蓋身分與註冊資訊、規模、流量、位址的使用地區、宣告的前綴及其 RPKI 狀態、互連關係，以及對等策略、接入的網際網路交換中心與機房駐點", + "fr": "Nouvel outil : Profil ASN — le profil complet de n'importe quel système autonome : identité et enregistrement, taille, trafic, pays où ses adresses sont utilisées, préfixes annoncés avec leur statut RPKI, interconnexions, ainsi que politique de peering, points d'échange internet et centres de données où il est présent", + "ru": "Новый инструмент: профиль ASN — полный профиль любой автономной системы: владелец и регистрация, масштаб, трафик, где используются её адреса, анонсируемые префиксы со статусом RPKI, связность, а также политика пиринга, точки обмена трафиком и дата-центры, где она присутствует", + "pt-BR": "Nova ferramenta: Perfil de ASN — o perfil completo de qualquer sistema autônomo: identidade e registro, porte, tráfego, onde seus endereços são usados, prefixos anunciados com status RPKI, conectividade, além da política de peering, pontos de troca de tráfego e data centers em que está presente" } }, { diff --git a/frontend/locales/en.json b/frontend/locales/en.json index 89c1ddc3..7e9bc830 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -1129,6 +1129,58 @@ "peers": "Peers", "customers": "Customers", "none": "None observed" + }, + "peering": { + "title": "Peering & facilities", + "desc": "As registered by the network's own operators in PeeringDB. Internet exchanges are the IXPs it connects to, with port capacity summed per exchange; RS marks a session with the exchange's route servers. Facilities are the data centres it is present in.", + "policy": "Peering policy", + "traffic": "Traffic level", + "ratio": "Traffic ratio", + "irr": "IRR as-set", + "website": "Website", + "scope": "Geographic scope", + "exchanges": "Internet exchanges ({count})", + "facilities": "Facilities ({count})", + "rs": "RS", + "rsTitle": "Peers with the exchange's route servers", + "showAll": "Show all {count}", + "showLess": "Show fewer", + "types": { + "isp": "Cable/DSL/ISP", + "nsp": "NSP", + "content": "Content", + "enterprise": "Enterprise", + "education": "Educational/Research", + "nonProfit": "Non-profit", + "government": "Government", + "routeServer": "Route server", + "routeCollector": "Route collector", + "networkServices": "Network services" + }, + "scopes": { + "regional": "Regional", + "northAmerica": "North America", + "asiaPacific": "Asia Pacific", + "europe": "Europe", + "southAmerica": "South America", + "africa": "Africa", + "australia": "Australia", + "middleEast": "Middle East", + "global": "Global" + }, + "ratios": { + "balanced": "Balanced", + "mostlyInbound": "Mostly inbound", + "heavyInbound": "Heavy inbound", + "mostlyOutbound": "Mostly outbound", + "heavyOutbound": "Heavy outbound" + }, + "policies": { + "open": "Open", + "selective": "Selective", + "restrictive": "Restrictive", + "no": "No peering" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/locales/fr.json b/frontend/locales/fr.json index 8898b100..b05bc83b 100644 --- a/frontend/locales/fr.json +++ b/frontend/locales/fr.json @@ -1129,6 +1129,58 @@ "peers": "Pairs", "customers": "Clients", "none": "Aucun observé" + }, + "peering": { + "title": "Peering et centres de données", + "desc": "Informations déclarées par les opérateurs du réseau eux-mêmes dans PeeringDB. Les points d'échange internet sont les IXP auxquels il est raccordé, avec la capacité des ports additionnée par point d'échange ; RS signale une session avec les serveurs de routes du point d'échange. Les centres de données sont ceux où il est présent.", + "policy": "Politique de peering", + "traffic": "Volume de trafic", + "ratio": "Ratio de trafic", + "irr": "as-set IRR", + "website": "Site web", + "scope": "Couverture géographique", + "exchanges": "Points d'échange internet ({count})", + "facilities": "Centres de données ({count})", + "rs": "RS", + "rsTitle": "Pair des serveurs de routes du point d'échange", + "showAll": "Tout afficher ({count})", + "showLess": "Afficher moins", + "types": { + "isp": "Câble/DSL/FAI", + "nsp": "NSP", + "content": "Contenu", + "enterprise": "Entreprise", + "education": "Enseignement/Recherche", + "nonProfit": "Non lucratif", + "government": "Administration", + "routeServer": "Serveur de routes", + "routeCollector": "Collecteur de routes", + "networkServices": "Services réseau" + }, + "scopes": { + "regional": "Régional", + "northAmerica": "Amérique du Nord", + "asiaPacific": "Asie-Pacifique", + "europe": "Europe", + "southAmerica": "Amérique du Sud", + "africa": "Afrique", + "australia": "Australie", + "middleEast": "Moyen-Orient", + "global": "Mondial" + }, + "ratios": { + "balanced": "Équilibré", + "mostlyInbound": "Plutôt entrant", + "heavyInbound": "Fortement entrant", + "mostlyOutbound": "Plutôt sortant", + "heavyOutbound": "Fortement sortant" + }, + "policies": { + "open": "Ouverte", + "selective": "Sélective", + "restrictive": "Restrictive", + "no": "Pas de peering" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/locales/pt-BR.json b/frontend/locales/pt-BR.json index b7795a7f..dba553af 100644 --- a/frontend/locales/pt-BR.json +++ b/frontend/locales/pt-BR.json @@ -1129,6 +1129,58 @@ "peers": "Peers", "customers": "Clientes", "none": "Nenhum observado" + }, + "peering": { + "title": "Peering e data centers", + "desc": "Informações declaradas pelos próprios operadores da rede no PeeringDB. Os pontos de troca de tráfego são os IXPs aos quais ela se conecta, com a capacidade das portas somada por ponto de troca; RS indica sessão com os servidores de rota do ponto de troca. Os data centers são aqueles em que ela está presente.", + "policy": "Política de peering", + "traffic": "Volume de tráfego", + "ratio": "Proporção de tráfego", + "irr": "as-set IRR", + "website": "Site", + "scope": "Abrangência", + "exchanges": "Pontos de troca de tráfego ({count})", + "facilities": "Data centers ({count})", + "rs": "RS", + "rsTitle": "Faz peering com os servidores de rota do ponto de troca", + "showAll": "Mostrar todos ({count})", + "showLess": "Mostrar menos", + "types": { + "isp": "Cabo/DSL/ISP", + "nsp": "NSP", + "content": "Conteúdo", + "enterprise": "Empresa", + "education": "Educação/Pesquisa", + "nonProfit": "Sem fins lucrativos", + "government": "Governo", + "routeServer": "Servidor de rotas", + "routeCollector": "Coletor de rotas", + "networkServices": "Serviços de rede" + }, + "scopes": { + "regional": "Regional", + "northAmerica": "América do Norte", + "asiaPacific": "Ásia-Pacífico", + "europe": "Europa", + "southAmerica": "América do Sul", + "africa": "África", + "australia": "Austrália", + "middleEast": "Oriente Médio", + "global": "Global" + }, + "ratios": { + "balanced": "Equilibrado", + "mostlyInbound": "Principalmente de entrada", + "heavyInbound": "Fortemente de entrada", + "mostlyOutbound": "Principalmente de saída", + "heavyOutbound": "Fortemente de saída" + }, + "policies": { + "open": "Aberta", + "selective": "Seletiva", + "restrictive": "Restritiva", + "no": "Sem peering" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/locales/ru.json b/frontend/locales/ru.json index 540f96ac..6ebc1a6e 100644 --- a/frontend/locales/ru.json +++ b/frontend/locales/ru.json @@ -1129,6 +1129,58 @@ "peers": "Пиры", "customers": "Клиенты", "none": "Не обнаружено" + }, + "peering": { + "title": "Пиринг и площадки", + "desc": "Сведения, которые операторы сети сами указали в PeeringDB. Точки обмена трафиком — IX, к которым подключена сеть, ёмкость портов суммирована по каждой IX; RS означает сессию с роут-серверами IX. Площадки — дата-центры, в которых она присутствует.", + "policy": "Политика пиринга", + "traffic": "Объём трафика", + "ratio": "Соотношение трафика", + "irr": "IRR as-set", + "website": "Сайт", + "scope": "География", + "exchanges": "Точки обмена трафиком ({count})", + "facilities": "Площадки ({count})", + "rs": "RS", + "rsTitle": "Пиринг с роут-серверами точки обмена", + "showAll": "Показать все ({count})", + "showLess": "Свернуть", + "types": { + "isp": "Кабель/DSL/ISP", + "nsp": "NSP", + "content": "Контент", + "enterprise": "Предприятие", + "education": "Образование/наука", + "nonProfit": "Некоммерческая", + "government": "Госструктура", + "routeServer": "Роут-сервер", + "routeCollector": "Коллектор маршрутов", + "networkServices": "Сетевые услуги" + }, + "scopes": { + "regional": "Региональная", + "northAmerica": "Северная Америка", + "asiaPacific": "Азиатско-Тихоокеанский регион", + "europe": "Европа", + "southAmerica": "Южная Америка", + "africa": "Африка", + "australia": "Австралия", + "middleEast": "Ближний Восток", + "global": "Глобальная" + }, + "ratios": { + "balanced": "Сбалансированное", + "mostlyInbound": "Преимущественно входящий", + "heavyInbound": "Почти только входящий", + "mostlyOutbound": "Преимущественно исходящий", + "heavyOutbound": "Почти только исходящий" + }, + "policies": { + "open": "Открытая", + "selective": "Выборочная", + "restrictive": "Ограниченная", + "no": "Без пиринга" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/locales/zh-TW.json b/frontend/locales/zh-TW.json index dccebc18..0e8ae067 100644 --- a/frontend/locales/zh-TW.json +++ b/frontend/locales/zh-TW.json @@ -1129,6 +1129,58 @@ "peers": "對等", "customers": "客戶", "none": "未觀察到" + }, + "peering": { + "title": "對等與駐點", + "desc": "以下資訊由該網路的營運者在 PeeringDB 中自行登記。網際網路交換中心是它接入的 IX，連接埠速率按每個 IX 合計；RS 表示與該 IX 的路由伺服器建立了工作階段。機房駐點是它進駐的資料中心。", + "policy": "對等策略", + "traffic": "流量規模", + "ratio": "流量方向", + "irr": "IRR as-set", + "website": "官網", + "scope": "涵蓋範圍", + "exchanges": "網際網路交換中心（{count}）", + "facilities": "機房駐點（{count}）", + "rs": "RS", + "rsTitle": "與該交換中心的路由伺服器對等", + "showAll": "顯示全部 {count} 個", + "showLess": "收合", + "types": { + "isp": "有線/DSL/ISP", + "nsp": "NSP（網路服務商）", + "content": "內容", + "enterprise": "企業", + "education": "教育/學研", + "nonProfit": "非營利", + "government": "政府", + "routeServer": "路由伺服器", + "routeCollector": "路由收集器", + "networkServices": "網路服務" + }, + "scopes": { + "regional": "區域", + "northAmerica": "北美", + "asiaPacific": "亞太", + "europe": "歐洲", + "southAmerica": "南美", + "africa": "非洲", + "australia": "澳洲", + "middleEast": "中東", + "global": "全球" + }, + "ratios": { + "balanced": "均衡", + "mostlyInbound": "以入站為主", + "heavyInbound": "大量入站", + "mostlyOutbound": "以出站為主", + "heavyOutbound": "大量出站" + }, + "policies": { + "open": "開放", + "selective": "選擇性", + "restrictive": "限制性", + "no": "不對等" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/locales/zh.json b/frontend/locales/zh.json index c64fdabd..307a5fbb 100644 --- a/frontend/locales/zh.json +++ b/frontend/locales/zh.json @@ -1129,6 +1129,58 @@ "peers": "对等", "customers": "客户", "none": "未观察到" + }, + "peering": { + "title": "对等与驻点", + "desc": "以下信息由该网络的运营者在 PeeringDB 中自行登记。互联网交换中心是它接入的 IX，端口速率按每个 IX 合计；RS 表示与该 IX 的路由服务器建立了会话。机房驻点是它入驻的数据中心。", + "policy": "对等策略", + "traffic": "流量规模", + "ratio": "流量方向", + "irr": "IRR as-set", + "website": "官网", + "scope": "覆盖范围", + "exchanges": "互联网交换中心（{count}）", + "facilities": "机房驻点（{count}）", + "rs": "RS", + "rsTitle": "与该交换中心的路由服务器对等", + "showAll": "显示全部 {count} 个", + "showLess": "收起", + "types": { + "isp": "有线/DSL/ISP", + "nsp": "NSP（网络服务商）", + "content": "内容", + "enterprise": "企业", + "education": "教育/科研", + "nonProfit": "非营利", + "government": "政府", + "routeServer": "路由服务器", + "routeCollector": "路由收集器", + "networkServices": "网络服务" + }, + "scopes": { + "regional": "区域", + "northAmerica": "北美", + "asiaPacific": "亚太", + "europe": "欧洲", + "southAmerica": "南美", + "africa": "非洲", + "australia": "澳大利亚", + "middleEast": "中东", + "global": "全球" + }, + "ratios": { + "balanced": "均衡", + "mostlyInbound": "以入站为主", + "heavyInbound": "大量入站", + "mostlyOutbound": "以出站为主", + "heavyOutbound": "大量出站" + }, + "policies": { + "open": "开放", + "selective": "选择性", + "restrictive": "限制性", + "no": "不对等" + } } }, "enhanceddnsleaktest": { diff --git a/frontend/utils/asn-profile-view.js b/frontend/utils/asn-profile-view.js index 9d287c7c..33d6ce25 100644 --- a/frontend/utils/asn-profile-view.js +++ b/frontend/utils/asn-profile-view.js @@ -2,7 +2,8 @@ // the one `/api/asn-profile` response (sections + `status`, see // common/asn-profile.js): which sections render, hero identity, key facts, // registration rows, customer cone, RPKI summary, prefix rows, country -// shares, neighbour groups and the reputation meters / verdict. +// shares, neighbour groups, the reputation meters / verdict and the +// PeeringDB hero line, facts and IX / facility lists. // // Relative imports keep this file importable from the Node test runner. import { parseCidr } from './ip-math.js'; @@ -41,7 +42,11 @@ const isObject = (value) => value !== null && typeof value === 'object' && !Arra /* ------------------------------------------------------------------ */ // Cache-buster: bump on response-shape changes (sent as `v=`). -export const ASN_PROFILE_VERSION = 3; +export const ASN_PROFILE_VERSION = 4; + +// Client budget for the one request: above the backend's largest +// per-section deadline (DEADLINES in common/asn-profile.js). +export const ASN_PROFILE_TIMEOUT_MS = 20000; export const asnProfileUrl = (asn) => `/api/asn-profile?asn=${asn}&v=${ASN_PROFILE_VERSION}`; @@ -60,6 +65,7 @@ const SECTION_SOURCE = { whois: 'RIR RDAP', rank: 'CAIDA AS Rank', reputation: 'IPCheck.ing', + peeringdb: 'PeeringDB', }; export const failedSources = (profile) => [...new Set(Object.entries(profile?.status || {}) @@ -274,3 +280,94 @@ export const proxyListedCount = (rep) => { export const VERDICT_TONE = { none: 'ok-fast', low: 'ok-fast', medium: 'ok-slow', high: 'fail' }; export const verdictLevel = (rep) => (rep?.found === true && rep.level in VERDICT_TONE ? rep.level : null); + +/* ------------------------------------------------------------------ */ +/* PeeringDB */ +/* ------------------------------------------------------------------ */ + +// PeeringDB's fixed vocabularies → locale key slugs under +// asnprofile.peering.*; a value outside them is shown as PeeringDB wrote it. +export const PEERING_ENUMS = { + types: { + 'Cable/DSL/ISP': 'isp', NSP: 'nsp', Content: 'content', Enterprise: 'enterprise', + 'Educational/Research': 'education', 'Non-Profit': 'nonProfit', Government: 'government', + 'Route Server': 'routeServer', 'Route Collector': 'routeCollector', 'Network Services': 'networkServices', + }, + scopes: { + Regional: 'regional', 'North America': 'northAmerica', 'Asia Pacific': 'asiaPacific', Europe: 'europe', + 'South America': 'southAmerica', Africa: 'africa', Australia: 'australia', 'Middle East': 'middleEast', + Global: 'global', + }, + ratios: { + Balanced: 'balanced', 'Mostly Inbound': 'mostlyInbound', 'Heavy Inbound': 'heavyInbound', + 'Mostly Outbound': 'mostlyOutbound', 'Heavy Outbound': 'heavyOutbound', + }, + policies: { Open: 'open', Selective: 'selective', Restrictive: 'restrictive', No: 'no' }, +}; + +// Locale key for a vocabulary value, or null when it isn't one. +export const peeringEnumKey = (group, value) => { + const slug = PEERING_ENUMS[group]?.[value]; + return slug ? `asnprofile.peering.${group}.${slug}` : null; +}; + +// An http(s) URL without userinfo as { href, label: host without "www." }; +// null otherwise. +export const websiteLink = (value) => { + if (typeof value !== 'string') return null; + try { + const url = new URL(value.trim()); + if (url.protocol !== 'http:' && url.protocol !== 'https:') return null; + if (url.username || url.password) return null; + return { href: url.href, label: url.hostname.replace(/^www\./i, '') }; + } catch { + return null; + } +}; + +// Hero additions: network types, website, scope — each only when present. +export const peeringHero = (pdb) => ({ + types: Array.isArray(pdb?.types) ? pdb.types.filter((type) => typeof type === 'string' && type) : [], + website: websiteLink(pdb?.website), + scope: typeof pdb?.scope === 'string' && pdb.scope ? pdb.scope : null, +}); + +// The card's fact rows in display order, empties dropped. `group` names +// the vocabulary a value is translated through. +export const peeringFacts = (pdb) => [ + { key: 'policy', value: pdb?.policy, group: 'policies' }, + { key: 'traffic', value: pdb?.traffic }, + { key: 'ratio', value: pdb?.ratio, group: 'ratios' }, + { key: 'irr', value: pdb?.irrAsSet, mono: true }, +].filter((fact) => typeof fact.value === 'string' && fact.value); + +// Port capacity in Mbps → '100M' / '10G' / '1.2T' (one decimal at most). +// `format` renders the number (the page passes a locale formatter). +export const formatPortSpeed = (mbps, format = (n) => String(n)) => { + const value = Number(mbps); + if (!Number.isFinite(value) || value <= 0) return ''; + const [amount, unit] = value >= 1e6 ? [value / 1e6, 'T'] : value >= 1000 ? [value / 1000, 'G'] : [value, 'M']; + return `${format(Math.round(amount * 10) / 10)}${unit}`; +}; + +const placeRow = (item) => ({ + name: item.name, + city: typeof item.city === 'string' ? item.city : '', + cc: pickCountryCode(item.country), +}); + +// IX rows in the backend's order (capacity first); unnamed entries dropped. +export const exchangeRows = (pdb) => (Array.isArray(pdb?.ixs) ? pdb.ixs : []) + .filter((item) => typeof item?.name === 'string' && item.name) + .map((item) => ({ ...placeRow(item), speed: Number(item.speed) || 0, rsPeer: item.rsPeer === true })); + +export const facilityRows = (pdb) => (Array.isArray(pdb?.facilities) ? pdb.facilities : []) + .filter((item) => typeof item?.name === 'string' && item.name) + .map(placeRow); + +// The Peering & facilities card needs at least one list entry; facts alone +// don't carry it. +export const hasPeeringCard = (pdb) => exchangeRows(pdb).length > 0 || facilityRows(pdb).length > 0; + +// First `first` rows unless expanded. +export const visibleRows = (rows, expanded, first = 10) => (expanded ? rows : rows.slice(0, first)); diff --git a/tests/api-handlers.test.js b/tests/api-handlers.test.js index aded7643..bacbe86f 100644 --- a/tests/api-handlers.test.js +++ b/tests/api-handlers.test.js @@ -23,6 +23,7 @@ import macCheckerHandler from '../api/mac-checker.js'; import githubStarsHandler from '../api/github-stars.js'; import personaEvaluateHandler from '../api/persona.js'; import asnProfileHandler from '../api/asn-profile.js'; +import { isPeeringdbLoaded } from '../common/peeringdb-db.js'; import updateAchievementHandler from '../api/update-user-achievement.js'; import ipcheckIngHandler from '../api/ipcheck-ing.js'; import { getSessionResult as dnsLeakGetResult } from '../api/dns-leak-test.js'; @@ -696,7 +697,8 @@ describe('ipcheck-ing handler', () => { // -- asn-profile handler --------------------------------------------------- // Composition and deadlines are covered in tests/asn-profile-aggregate.test.js; // here: unconfigured sources read as disabled, failing upstreams as error, -// and the private-API pass-through for the reputation section. +// the local PeeringDB section, and the private-API pass-through for the +// reputation section. describe('asn-profile handler', () => { it('answers 200 with per-section statuses when only some sources work', async () => { @@ -718,6 +720,10 @@ describe('asn-profile handler', () => { assert.ok(['ok', 'empty'].includes(res.body.status.connectivity)); assert.equal(res.body.whois, null); assert.equal(res.body.rank, null); + // Local and never an error: disabled without an index file (the + // checkout default), ok / empty once the updater has built one. + if (isPeeringdbLoaded()) assert.ok(['ok', 'empty'].includes(res.body.status.peeringdb)); + else assert.equal(res.body.status.peeringdb, 'disabled'); }); it('forwards the caller headers to the private API for reputation', async () => { diff --git a/tests/asn-profile-aggregate.test.js b/tests/asn-profile-aggregate.test.js index 7e5ab1df..3be38ec8 100644 --- a/tests/asn-profile-aggregate.test.js +++ b/tests/asn-profile-aggregate.test.js @@ -1,20 +1,31 @@ // Tests for common/asn-profile.js — the /api/asn-profile composition: // per-section classification, deadlines, the all-failed verdict and the // edge-cache veto. Loaders are injected; nothing touches the network. +// The frontend's request budget comes from frontend/utils/asn-profile-view.js. import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { - SECTIONS, DEADLINES, withDeadline, hasMeaningfulField, - classifyRadar, classifyPrefixes, classifyConnectivity, classifyRank, classifyReputation, + SECTIONS, DEADLINES, INNER_TIMEOUTS, SOURCE_TIMEOUTS, withDeadline, hasMeaningfulField, + classifyRadar, classifyPrefixes, classifyConnectivity, classifyRank, classifyReputation, classifyPeeringdb, buildSectionLoaders, composeAsnProfile, allSourcesFailed, isCompleteProfile, } from '../common/asn-profile.js'; +import { ASN_PROFILE_TIMEOUT_MS } from '../frontend/utils/asn-profile-view.js'; const sleep = (ms, value) => new Promise((resolve) => setTimeout(() => resolve(value), ms)); describe('deadlines', () => { - it('every section has a budget and the worst case stays under 8 s', () => { - for (const section of SECTIONS) assert.ok(DEADLINES[section] > 0 && DEADLINES[section] <= 8000, section); + it('every deadline sits above the inner timeouts of the source it wraps', () => { + for (const section of SECTIONS) { + assert.ok(Number.isFinite(INNER_TIMEOUTS[section]), section); + assert.ok(DEADLINES[section] > INNER_TIMEOUTS[section], section); + } + assert.equal(INNER_TIMEOUTS.whois, 8000 + SOURCE_TIMEOUTS.autnum, 'IANA bootstrap, then autnum'); + }); + + it('the page waits longer than the slowest section, with headroom', () => { + const slowest = Math.max(...SECTIONS.map((section) => DEADLINES[section])); + assert.ok(ASN_PROFILE_TIMEOUT_MS >= slowest + 5000, `${ASN_PROFILE_TIMEOUT_MS} vs ${slowest}`); }); it('withDeadline passes a fast result through and rejects a slow one', async () => { @@ -58,6 +69,13 @@ describe('classification', () => { assert.deepEqual(classifyReputation({ status: 200, data: found }), { status: 'ok', data: found }); assert.throws(() => classifyReputation({ status: 503, data: { error: 'ASN data not loaded' } })); }); + + it('peeringdb: no index is disabled, no record empty, a record ok', () => { + assert.deepEqual(classifyPeeringdb(false, null), { status: 'disabled', data: null }); + assert.deepEqual(classifyPeeringdb(true, null), { status: 'empty', data: null }); + const record = { policy: 'Open', types: [], ixs: [], facilities: [] }; + assert.deepEqual(classifyPeeringdb(true, record), { status: 'ok', data: record }); + }); }); const deps = (overrides = {}) => ({ @@ -69,6 +87,8 @@ const deps = (overrides = {}) => ({ isAutnumMissing: (err) => err.message.startsWith('ASN not found'), queryAsRank: async () => ({ rank: 10 }), requestReputation: async () => null, + isPeeringdbLoaded: () => true, + lookupPeeringdb: (asn) => (asn === 13335 ? { policy: 'Open', types: ['Content'], ixs: [], facilities: [] } : null), ...overrides, }); @@ -91,9 +111,10 @@ describe('composeAsnProfile', () => { it('assembles every section with its status', async () => { const body = await composeAsnProfile(13335, buildSectionLoaders(deps())); assert.deepEqual(body.status, { - radar: 'ok', prefixes: 'ok', connectivity: 'ok', whois: 'ok', rank: 'ok', reputation: 'disabled', + radar: 'ok', prefixes: 'ok', connectivity: 'ok', whois: 'ok', rank: 'ok', reputation: 'disabled', peeringdb: 'ok', }); assert.equal(body.asn, 13335); + assert.equal(body.peeringdb.policy, 'Open'); assert.equal(body.radar.asnName, 'EXAMPLE'); assert.equal(body.reputation, null); assert.equal(isCompleteProfile(body), true); @@ -115,6 +136,16 @@ describe('composeAsnProfile', () => { assert.equal(isCompleteProfile(body), false); }); + it('peeringdb never makes an answer incomplete or failed', async () => { + const missing = await composeAsnProfile(64500, buildSectionLoaders(deps())); + assert.equal(missing.status.peeringdb, 'empty'); + assert.equal(isCompleteProfile(missing), true); + const unloaded = await composeAsnProfile(13335, buildSectionLoaders(deps({ isPeeringdbLoaded: () => false }))); + assert.equal(unloaded.status.peeringdb, 'disabled'); + assert.equal(unloaded.peeringdb, null); + assert.equal(isCompleteProfile(unloaded), true); + }); + it('a loader that throws synchronously is contained', async () => { const loaders = { ...buildSectionLoaders(deps()), whois: () => { throw new Error('boom'); } }; const body = await composeAsnProfile(1, loaders); diff --git a/tests/asn-profile-view.test.js b/tests/asn-profile-view.test.js index 5f3d5fd9..f5955b35 100644 --- a/tests/asn-profile-view.test.js +++ b/tests/asn-profile-view.test.js @@ -1,8 +1,8 @@ // Tests for frontend/utils/asn-profile-view.js — the ASN Profile page's // pure shaping over the one /api/asn-profile response: which sections // render, hero identity and key facts, registration rows, customer cone, -// RPKI summary, prefix rows, country shares, neighbour groups and the -// reputation meters. +// RPKI summary, prefix rows, country shares, neighbour groups, the +// reputation meters and the PeeringDB hero / facts / lists. import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; @@ -14,7 +14,10 @@ import { normalizeRpki, shapePrefixRows, familyCounts, filterPrefixRows, rpkiSummary, prefixListText, countryShareRows, neighbourGroups, ratioPosition, BASELINE_POSITION, reputationMeters, proxyListedCount, verdictLevel, VERDICT_TONE, + PEERING_ENUMS, peeringEnumKey, websiteLink, peeringHero, peeringFacts, formatPortSpeed, + exchangeRows, facilityRows, hasPeeringCard, visibleRows, } from '../frontend/utils/asn-profile-view.js'; +import en from '../frontend/locales/en.json' with { type: 'json' }; describe('parseCount', () => { it('reads server-formatted integers', () => { @@ -303,3 +306,87 @@ describe('reputation', () => { assert.equal(verdictLevel({ ...rep, level: null }), null); }); }); + +describe('PeeringDB', () => { + const pdb = { + website: 'https://www.example.net/about', + types: ['Content', 'NSP'], + scope: 'Global', + traffic: '1-5Tbps', + ratio: 'Mostly Outbound', + policy: 'Open', + irrAsSet: 'AS-EXAMPLE', + ixs: [ + { name: 'IX.br (PTT.br) São Paulo', city: 'São Paulo/SP', country: 'BR', speed: 1800000, rsPeer: true }, + { name: 'DE-CIX Frankfurt', city: 'Frankfurt', country: 'de', speed: 100000, rsPeer: false }, + { name: '', city: 'x', country: 'US', speed: 1 }, + ], + facilities: [{ name: 'Equinix FR5', city: null, country: 'XX1' }], + }; + + it('maps every PeeringDB vocabulary value to an English label', () => { + for (const [group, values] of Object.entries(PEERING_ENUMS)) { + for (const value of Object.keys(values)) { + const key = peeringEnumKey(group, value); + const label = key.split('.').reduce((node, part) => node?.[part], en); + assert.equal(typeof label, 'string', key); + } + } + assert.equal(peeringEnumKey('types', 'Something New'), null); + assert.equal(peeringEnumKey('nope', 'Open'), null); + }); + + it('website links are http(s) only, labelled by host', () => { + assert.deepEqual(websiteLink('https://www.example.net/about'), { href: 'https://www.example.net/about', label: 'example.net' }); + assert.equal(websiteLink('http://ripe.net').label, 'ripe.net'); + for (const bad of ['javascript:alert(1)', 'ftp://x.example', 'http://noc.isp@mail.example', 'not a url', null]) { + assert.equal(websiteLink(bad), null, String(bad)); + } + }); + + it('hero carries only what is present', () => { + assert.deepEqual(peeringHero(pdb), { types: ['Content', 'NSP'], website: websiteLink(pdb.website), scope: 'Global' }); + assert.deepEqual(peeringHero(null), { types: [], website: null, scope: null }); + }); + + it('facts keep display order and drop empties', () => { + assert.deepEqual(peeringFacts(pdb).map((fact) => fact.key), ['policy', 'traffic', 'ratio', 'irr']); + assert.deepEqual(peeringFacts({ policy: 'Open', ratio: null }), [{ key: 'policy', value: 'Open', group: 'policies' }]); + assert.deepEqual(peeringFacts(null), []); + }); + + it('formats port capacity as M / G / T', () => { + assert.equal(formatPortSpeed(100), '100M'); + assert.equal(formatPortSpeed(1000), '1G'); + assert.equal(formatPortSpeed(10000), '10G'); + assert.equal(formatPortSpeed(2500), '2.5G'); + assert.equal(formatPortSpeed(1800000), '1.8T'); + assert.equal(formatPortSpeed(1234567), '1.2T'); + assert.equal(formatPortSpeed(0), ''); + assert.equal(formatPortSpeed(400000, (n) => `<${n}>`), '<400>G'); + }); + + it('list rows: unnamed entries dropped, country codes checked, city as written', () => { + assert.deepEqual(exchangeRows(pdb), [ + { name: 'IX.br (PTT.br) São Paulo', city: 'São Paulo/SP', cc: 'BR', speed: 1800000, rsPeer: true }, + { name: 'DE-CIX Frankfurt', city: 'Frankfurt', cc: 'DE', speed: 100000, rsPeer: false }, + ]); + assert.deepEqual(facilityRows(pdb), [{ name: 'Equinix FR5', city: '', cc: null }]); + assert.deepEqual(exchangeRows(null), []); + }); + + it('the card shows only when a list has an entry', () => { + assert.equal(hasPeeringCard(pdb), true); + assert.equal(hasPeeringCard({ facilities: [{ name: 'F' }] }), true); + assert.equal(hasPeeringCard({ policy: 'Open', irr: 'AS-EXAMPLE' }), false); + assert.equal(hasPeeringCard({ website: 'https://x.example', types: ['NSP'], scope: 'Global' }), false); + assert.equal(hasPeeringCard(null), false); + }); + + it('lists show the first rows until expanded', () => { + const rows = Array.from({ length: 25 }, (_, i) => i); + assert.equal(visibleRows(rows, false).length, 10); + assert.equal(visibleRows(rows, true).length, 25); + assert.equal(visibleRows(rows.slice(0, 4), false).length, 4); + }); +}); diff --git a/tests/caida-updater.test.js b/tests/caida-updater.test.js new file mode 100644 index 00000000..c7e8d480 --- /dev/null +++ b/tests/caida-updater.test.js @@ -0,0 +1,182 @@ +// Tests for common/caida-updater.js — the PeeringDB row and the two +// capabilities it adds (no decompression; a distill step), the Cloudflare-key +// gate, and that a decompressing row still publishes as before. Every row is +// re-pointed at a temp dir and `fetch` is stubbed: no network, nothing +// written into the repo's dataset directories. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import { after, afterEach, describe, it } from 'node:test'; + +import { + datasets, updateDataset, bootstrapDataset, findPeeringdbDump, isDatasetEnabled, +} from '../common/caida-updater.js'; +import { PEERINGDB_FILE, readPeeringdbIndex, expandNet } from '../common/peeringdb-db.js'; + +const realFetch = globalThis.fetch; +const savedEnv = { key: process.env.CLOUDFLARE_API_KEY, legacy: process.env.CLOUDFLARE_API }; +afterEach(() => { + globalThis.fetch = realFetch; + for (const [name, value] of [['CLOUDFLARE_API_KEY', savedEnv.key], ['CLOUDFLARE_API', savedEnv.legacy]]) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'myip-caida-test-')); +after(() => fs.rmSync(root, { recursive: true, force: true })); + +const row = (id) => datasets.find((dataset) => dataset.id === id); + +// A row re-pointed at its own temp dir, with a recording reload. +const tempRow = (id, overrides = {}) => { + const reloads = []; + const dataset = { + ...row(id), + dbDir: fs.mkdtempSync(path.join(root, `${id}-`)), + reload: (reason) => reloads.push(reason), + ...overrides, + }; + return { dataset, reloads }; +}; + +// Minimal PeeringDB dump with a contact table beside the four wanted ones. +const DUMP = JSON.stringify({ + poc: { data: [{ net_id: 1, email: 'noc@example.net' }], meta: {} }, + net: { data: [{ id: 1, asn: 64500, policy_general: 'Open', info_types: ['NSP'], status: 'ok' }], meta: {} }, + ix: { data: [{ id: 10, name: 'AMS-IX', city: 'Amsterdam', country: 'NL', status: 'ok' }], meta: {} }, + netixlan: { data: [{ asn: 64500, ix_id: 10, speed: 100000, is_rs_peer: true, operational: true, status: 'ok' }], meta: {} }, + netfac: { data: [{ net_id: 1, fac_id: 20, name: 'NIKHEF', city: 'Amsterdam', country: 'NL', status: 'ok' }], meta: {} }, +}); + +const LISTING = `md5.md5 +peeringdb_2_dump_2026_10_01.json +peeringdb_2_dump_2026_10_02.json`; + +const BASE = 'https://publicdata.caida.org/datasets/peeringdb/'; + +// fetch stub over a URL → [status, body] map; records what was asked. +const stubFetch = (routes) => { + const requested = []; + globalThis.fetch = async (url) => { + requested.push(String(url)); + const [status, body] = routes[String(url)] || [404, 'not found']; + return new Response(body, { status }); + }; + return requested; +}; + +const peeringdbRoutes = () => ({ + [`${BASE}2026/10/`]: [200, LISTING], + [`${BASE}2026/10/peeringdb_2_dump_2026_10_02.json`]: [200, DUMP], +}); + +describe('findPeeringdbDump', () => { + it('takes the newest dump of the current UTC month', async () => { + stubFetch(peeringdbRoutes()); + const remote = await findPeeringdbDump({ now: new Date('2026-10-20T00:00:00Z') }); + assert.deepEqual(remote, { + url: `${BASE}2026/10/peeringdb_2_dump_2026_10_02.json`, + identifier: 'peeringdb_2_dump_2026_10_02.json', + }); + }); + + it('falls back to the previous month before the first dump lands', async () => { + const requested = stubFetch({ ...peeringdbRoutes(), [`${BASE}2026/11/`]: [200, 'empty'] }); + const remote = await findPeeringdbDump({ now: new Date('2026-11-01T03:00:00Z') }); + assert.equal(remote.identifier, 'peeringdb_2_dump_2026_10_02.json'); + assert.deepEqual(requested, [`${BASE}2026/11/`, `${BASE}2026/10/`]); + // Missing month directory (404) also falls back, across a year boundary. + stubFetch({ [`${BASE}2025/12/`]: [200, LISTING.replaceAll('2026_10', '2025_12')] }); + assert.equal((await findPeeringdbDump({ now: new Date('2026-01-01T00:00:00Z') })).identifier, + 'peeringdb_2_dump_2025_12_02.json'); + }); + + it('fails on a server error or when neither month has a dump', async () => { + stubFetch({ [`${BASE}2026/10/`]: [503, 'busy'] }); + await assert.rejects(findPeeringdbDump({ now: new Date('2026-10-20T00:00:00Z') }), /HTTP 503/); + stubFetch({}); + await assert.rejects(findPeeringdbDump({ now: new Date('2026-10-20T00:00:00Z') }), /No PeeringDB dump/); + }); +}); + +describe('peeringdb row', () => { + it('is gated on the Cloudflare key; the CAIDA rows are not', () => { + delete process.env.CLOUDFLARE_API_KEY; + delete process.env.CLOUDFLARE_API; + assert.equal(isDatasetEnabled(row('peeringdb')), false); + assert.equal(isDatasetEnabled(row('as2org')), true); + assert.equal(isDatasetEnabled(row('as-rel')), true); + process.env.CLOUDFLARE_API_KEY = 'k'; + assert.equal(isDatasetEnabled(row('peeringdb')), true); + }); + + it('downloads without decompressing, distills, publishes only the index', async () => { + stubFetch(peeringdbRoutes()); + const { dataset, reloads } = tempRow('peeringdb', { + findRemote: (opts) => findPeeringdbDump({ ...opts, now: new Date('2026-10-20T00:00:00Z') }), + validate: async () => {}, // the real floor (20k networks) is tested below + }); + const result = await updateDataset(dataset); + assert.deepEqual(result, { updated: true, identifier: 'peeringdb_2_dump_2026_10_02.json' }); + assert.deepEqual(reloads, ['auto update']); + + const files = fs.readdirSync(dataset.dbDir).sort(); + assert.deepEqual(files, ['.caida-update-state.json', PEERINGDB_FILE]); + const published = fs.readFileSync(path.join(dataset.dbDir, PEERINGDB_FILE), 'utf8'); + assert.equal(published.includes('@'), false, 'no contact data'); + const record = expandNet(readPeeringdbIndex(path.join(dataset.dbDir, PEERINGDB_FILE)), 64500); + assert.deepEqual(record.ixs, [{ name: 'AMS-IX', city: 'Amsterdam', country: 'NL', speed: 100000, rsPeer: true }]); + + // Same dump again → nothing to do. + assert.deepEqual(await updateDataset(dataset), { updated: false, reason: 'not-modified' }); + }); + + it('refuses to publish an index below the network floor', async () => { + stubFetch(peeringdbRoutes()); + const { dataset, reloads } = tempRow('peeringdb', { + findRemote: (opts) => findPeeringdbDump({ ...opts, now: new Date('2026-10-20T00:00:00Z') }), + }); + await assert.rejects(updateDataset(dataset), /only 1 networks/); + assert.deepEqual(fs.readdirSync(dataset.dbDir), []); + assert.deepEqual(reloads, []); + }); + + it('reports a dump that cannot be distilled', async () => { + stubFetch({ ...peeringdbRoutes(), [`${BASE}2026/10/peeringdb_2_dump_2026_10_02.json`]: [200, DUMP.slice(0, 200)] }); + const { dataset } = tempRow('peeringdb', { + findRemote: (opts) => findPeeringdbDump({ ...opts, now: new Date('2026-10-20T00:00:00Z') }), + }); + await assert.rejects(updateDataset(dataset), /peeringdb distill phase failed: truncated/); + }); + + it('bootstrap skips a present index but not a directory holding only state files', async () => { + const requested = stubFetch({}); + const { dataset } = tempRow('peeringdb'); + fs.writeFileSync(path.join(dataset.dbDir, '.caida-update-state.json'), '{}'); + assert.equal((await bootstrapDataset(dataset)).status, 'failed'); + assert.ok(requested.length > 0, 'a state file alone is not a snapshot'); + + requested.length = 0; + fs.writeFileSync(path.join(dataset.dbDir, PEERINGDB_FILE), '{}'); + assert.equal((await bootstrapDataset(dataset)).status, 'present'); + assert.deepEqual(requested, []); + }); +}); + +describe('decompressing rows', () => { + it('still download, decompress and publish the text snapshot', async () => { + const text = '# as2org fixture\n1|20260101|Example|US|ARIN\n'; + stubFetch({ 'https://example.invalid/latest.txt.gz': [200, zlib.gzipSync(text)] }); + const { dataset, reloads } = tempRow('as2org', { + findRemote: async () => ({ url: 'https://example.invalid/latest.txt.gz', identifier: 'v1' }), + validate: async () => {}, + }); + assert.deepEqual(await updateDataset(dataset), { updated: true, identifier: 'v1' }); + assert.equal(fs.readFileSync(path.join(dataset.dbDir, dataset.canonicalFile), 'utf8'), text); + assert.deepEqual(reloads, ['auto update']); + }); +}); diff --git a/tests/peeringdb-db.test.js b/tests/peeringdb-db.test.js new file mode 100644 index 00000000..2c5dcc9c --- /dev/null +++ b/tests/peeringdb-db.test.js @@ -0,0 +1,190 @@ +// Tests for common/peeringdb-db.js — the pure distill from projected dump +// rows to the per-ASN index (IX join, non-operational ports dropped, port +// speeds summed per IX, "Not Disclosed" as empty, multiple network types, +// no contact data), and reading an index file back into lookups. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, describe, it, mock } from 'node:test'; + +import { + PEERINGDB_DB_DIR, PEERINGDB_FILE, PEERINGDB_INDEX_VERSION, cleanText, projectRow, buildPeeringdbIndex, + readPeeringdbIndex, expandNet, reloadPeeringdbDatabase, isPeeringdbLoaded, lookupPeeringdb, +} from '../common/peeringdb-db.js'; + +// Raw rows as the dump has them (only a few of their many fields). +const RAW = { + net: [ + { + id: 1, asn: 64500, name: 'Example Net', website: 'https://www.example.net', + info_types: ['Content', 'NSP'], info_scope: 'Global', info_traffic: '1-5Tbps', + info_ratio: 'Not Disclosed', policy_general: 'Open', irr_as_set: 'AS-EXAMPLE', + looking_glass: 'https://lg.example.net', policy_url: 'https://example.net/peering', + route_server: 'rs.example.net', status: 'ok', + }, + // Only undisclosed / empty fields and no presence → dropped. + { id: 2, asn: 64501, name: 'Quiet Net', info_types: [], info_scope: 'Not Disclosed', info_ratio: '', website: '', status: 'ok' }, + // Facilities only; a non-http website is dropped. + { id: 3, asn: 64502, name: 'Fac Net', website: 'javascript:alert(1)', info_types: ['Enterprise'], status: 'ok' }, + // An e-mail address typed as a website (URL userinfo) is dropped too. + { id: 5, asn: 64504, name: 'Mail Net', website: 'http://noc.isp@mail.example', policy_general: 'Open', status: 'ok' }, + // Deleted record → ignored. + { id: 4, asn: 64503, name: 'Gone', policy_general: 'Open', status: 'deleted' }, + ], + ix: [ + { id: 10, name: 'DE-CIX Frankfurt', city: 'Frankfurt', country: 'DE', status: 'ok' }, + { id: 11, name: 'IX.br (PTT.br) São Paulo', city: 'São Paulo/SP', country: 'BR', status: 'ok' }, + { id: 12, name: 'Unused IX', city: 'Nowhere', country: 'US', status: 'ok' }, + ], + netixlan: [ + // Two ports at DE-CIX: speeds sum, the RS flag sticks. + { net_id: 1, ix_id: 10, asn: 64500, speed: 100000, is_rs_peer: false, operational: true, status: 'ok' }, + { net_id: 1, ix_id: 10, asn: 64500, speed: 100000, is_rs_peer: true, operational: true, status: 'ok' }, + { net_id: 1, ix_id: 11, asn: 64500, speed: 400000, is_rs_peer: false, operational: true, status: 'ok' }, + // Non-operational port and unknown IX are dropped. + { net_id: 1, ix_id: 11, asn: 64500, speed: 100000, is_rs_peer: true, operational: false, status: 'ok' }, + { net_id: 1, ix_id: 99, asn: 64500, speed: 100000, is_rs_peer: false, operational: true, status: 'ok' }, + ], + netfac: [ + { net_id: 1, fac_id: 20, name: 'Equinix FR5', city: 'Frankfurt', country: 'DE', status: 'ok' }, + { net_id: 1, fac_id: 21, name: 'Equinix SP4', city: 'São Paulo', country: 'BR', status: 'ok' }, + { net_id: 3, fac_id: 20, name: 'Equinix FR5', city: 'Frankfurt', country: 'DE', status: 'ok' }, + { net_id: 3, fac_id: 22, name: 'Old Site', city: 'Paris', country: 'FR', status: 'deleted' }, + ], + // Contact data the projection must never carry. + poc: [{ net_id: 1, email: 'noc@example.net', phone: '+1 555 0100' }], +}; + +const project = (raw) => Object.fromEntries( + ['net', 'ix', 'netixlan', 'netfac'].map((table) => [table, raw[table].map(projectRow[table])]), +); + +const index = buildPeeringdbIndex(project(RAW)); + +describe('cleanText', () => { + it('treats empty and "Not Disclosed" as absent', () => { + assert.equal(cleanText(' Open '), 'Open'); + assert.equal(cleanText('Not Disclosed'), undefined); + assert.equal(cleanText('not disclosed'), undefined); + assert.equal(cleanText(''), undefined); + assert.equal(cleanText(null), undefined); + }); +}); + +describe('buildPeeringdbIndex', () => { + it('keeps the profile fields, drops undisclosed and unused ones', () => { + const net = index.nets[64500]; + assert.equal(index.v, PEERINGDB_INDEX_VERSION); + assert.equal(net.website, 'https://www.example.net'); + assert.deepEqual(net.types, ['Content', 'NSP']); + assert.equal(net.scope, 'Global'); + assert.equal(net.traffic, '1-5Tbps'); + assert.equal(net.policy, 'Open'); + assert.equal(net.irr, 'AS-EXAMPLE'); + assert.equal('ratio' in net, false); + for (const dropped of ['looking_glass', 'policy_url', 'route_server', 'name']) assert.equal(dropped in net, false, dropped); + }); + + it('joins IX names from the ix table, sums ports, skips non-operational ones', () => { + // Capacity first: São Paulo 400G, then DE-CIX 2 × 100G with RS. + assert.deepEqual(index.nets[64500].ix, [[11, 400000, 0], [10, 200000, 1]]); + assert.deepEqual(index.ix[10], ['DE-CIX Frankfurt', 'Frankfurt', 'DE']); + assert.deepEqual(index.ix[11], ['IX.br (PTT.br) São Paulo', 'São Paulo/SP', 'BR']); + assert.equal(index.ix[12], undefined, 'unreferenced IX not stored'); + }); + + it('lists live facilities by country, then city', () => { + assert.deepEqual(index.nets[64500].fac, [21, 20]); + assert.deepEqual(index.nets[64502].fac, [20]); + assert.equal(index.fac[22], undefined); + }); + + it('drops networks with nothing to show and deleted records', () => { + assert.equal(index.nets[64501], undefined); + assert.equal(index.nets[64503], undefined); + assert.equal(index.nets[64502].website, undefined, 'non-http website dropped'); + assert.deepEqual(index.nets[64504], { policy: 'Open' }, 'userinfo website dropped'); + }); + + it('carries no contact data', () => { + const text = JSON.stringify(index); + assert.equal(text.includes('@'), false); + assert.equal(text.includes('555'), false); + }); +}); + +describe('readPeeringdbIndex / expandNet', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'myip-pdb-test-')); + after(() => fs.rmSync(dir, { recursive: true, force: true })); + + it('round-trips an index file into the API record', () => { + const file = path.join(dir, 'index.json'); + fs.writeFileSync(file, JSON.stringify(index)); + const tables = readPeeringdbIndex(file); + assert.equal(tables.nets.size, 3); + assert.deepEqual(expandNet(tables, 64500), { + website: 'https://www.example.net', + types: ['Content', 'NSP'], + scope: 'Global', + traffic: '1-5Tbps', + ratio: null, + policy: 'Open', + irrAsSet: 'AS-EXAMPLE', + ixs: [ + { name: 'IX.br (PTT.br) São Paulo', city: 'São Paulo/SP', country: 'BR', speed: 400000, rsPeer: false }, + { name: 'DE-CIX Frankfurt', city: 'Frankfurt', country: 'DE', speed: 200000, rsPeer: true }, + ], + facilities: [ + { name: 'Equinix SP4', city: 'São Paulo', country: 'BR' }, + { name: 'Equinix FR5', city: 'Frankfurt', country: 'DE' }, + ], + }); + assert.equal(expandNet(tables, '64502').types[0], 'Enterprise'); + assert.equal(expandNet(tables, 64501), null); + assert.equal(expandNet(null, 64500), null); + }); + + it('rejects a file of another version', () => { + const file = path.join(dir, 'old.json'); + fs.writeFileSync(file, JSON.stringify({ nets: {} })); + assert.throws(() => readPeeringdbIndex(file), /unsupported/); + }); +}); + +// The module's own file, faked through `fs` (as tests/as-rel-db.test.js +// does), so the result holds whether or not this checkout has an index. +describe('module state', () => { + const INDEX_PATH = path.join(PEERINGDB_DB_DIR, PEERINGDB_FILE); + const real = { existsSync: fs.existsSync, readFileSync: fs.readFileSync }; + const serve = (content) => { + mock.method(fs, 'existsSync', (p, ...rest) => (p === INDEX_PATH ? content !== null : real.existsSync(p, ...rest))); + mock.method(fs, 'readFileSync', (p, ...rest) => (p === INDEX_PATH ? content : real.readFileSync(p, ...rest))); + reloadPeeringdbDatabase('test'); + }; + after(() => { + mock.restoreAll(); + reloadPeeringdbDatabase('test cleanup'); + }); + + it('without an index file nothing is loaded and lookups are null', () => { + serve(null); + assert.equal(isPeeringdbLoaded(), false); + assert.equal(lookupPeeringdb(64500), null); + }); + + it('serves records from the index file', () => { + serve(JSON.stringify(index)); + assert.equal(isPeeringdbLoaded(), true); + assert.equal(lookupPeeringdb(64500).policy, 'Open'); + assert.equal(lookupPeeringdb(64501), null); + }); + + it('a corrupt file keeps the previous index', () => { + serve('{"v":1,"nets":'); + assert.equal(isPeeringdbLoaded(), true); + assert.equal(lookupPeeringdb(64500).policy, 'Open'); + mock.restoreAll(); + }); +}); diff --git a/tests/peeringdb-distill.test.js b/tests/peeringdb-distill.test.js new file mode 100644 index 00000000..157497d7 --- /dev/null +++ b/tests/peeringdb-distill.test.js @@ -0,0 +1,154 @@ +// Tests for common/peeringdb-distill.js — the streaming dump scanner and the +// batched index writer, over synthetic dumps written to a temp dir: only the +// four wanted tables are read (contact tables never reach the index), rows +// and keys straddling the 1 MB read boundary, escaped quotes / brackets / +// UTF-8 inside strings, scalar values beside table keys, and the refusals +// (truncated dump, missing table, abort). + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, describe, it } from 'node:test'; + +import { scanDumpRows, writeIndexFile, distillPeeringdbDump } from '../common/peeringdb-distill.js'; +import { buildPeeringdbIndex, projectRow, readPeeringdbIndex, expandNet } from '../common/peeringdb-db.js'; + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'myip-pdb-distill-')); +after(() => fs.rmSync(dir, { recursive: true, force: true })); + +const CHUNK = 1 << 20; + +// The four wanted tables (plus a scalar beside a table key). +const tables = () => ({ + net: { + generated: 'string value at table level', + data: [ + { + id: 1, asn: 64500, name: 'Example "Net" {a} [b]', website: 'https://example.net', + info_types: ['Content'], info_scope: 'Europe', info_traffic: '1-5Gbps', + info_ratio: 'Balanced', policy_general: 'Selective', irr_as_set: 'AS-EX', status: 'ok', + notes: 'escaped \\" quote and } brace', + }, + { id: 2, asn: 64501, name: 'Second', policy_general: 'Open', status: 'ok' }, + ], + meta: { note: 'ignored' }, + }, + ix: { data: [{ id: 10, name: 'IX "Quoted" São Paulo', city: 'São Paulo/SP', country: 'BR', status: 'ok' }], meta: {} }, + netixlan: { + data: [ + { net_id: 1, ix_id: 10, asn: 64500, speed: 10000, is_rs_peer: true, operational: true, status: 'ok' }, + { net_id: 1, ix_id: 10, asn: 64500, speed: 10000, is_rs_peer: false, operational: true, status: 'ok' }, + ], + meta: {}, + }, + netfac: { data: [{ net_id: 2, fac_id: 20, name: 'Fac [1]', city: 'Zürich', country: 'CH', status: 'ok' }], meta: {} }, +}); + +// A dump as CAIDA serves it: other tables (incl. contacts) around the four. +const dumpText = ({ omit, padBefore = '' } = {}) => { + const wanted = tables(); + if (omit) delete wanted[omit]; + const body = { + version: '2', + api: { data: [{ net: 'https://www.peeringdb.com/api/net' }], meta: {} }, + org: { data: [{ id: 1, name: 'Org', address1: 'Main Street 1', phone: '+1 555 0100' }], meta: {} }, + poc: { data: [{ net_id: 1, email: 'noc@example.net', phone: '+1 555 0199' }], meta: {} }, + ...wanted, + }; + const text = JSON.stringify(body, null, 1); + return padBefore ? `{"pad": {"data": [{"x": "${padBefore}"}], "meta": {}},${text.slice(1)}` : text; +}; + +const writeDump = (name, text) => { + const file = path.join(dir, name); + fs.writeFileSync(file, text); + return file; +}; + +const collect = async (file, wanted = new Set(['net', 'ix', 'netixlan', 'netfac'])) => { + const seen = []; + await scanDumpRows(file, wanted, (table, row) => seen.push([table, row])); + return seen; +}; + +describe('scanDumpRows', () => { + it('yields the wanted tables\' rows only, parsed intact', async () => { + const seen = await collect(writeDump('plain.json', dumpText())); + assert.deepEqual(seen.map(([table]) => table), ['net', 'net', 'ix', 'netixlan', 'netixlan', 'netfac']); + assert.equal(seen[0][1].name, 'Example "Net" {a} [b]'); + assert.equal(seen[0][1].notes, 'escaped \\" quote and } brace'); + assert.equal(seen[2][1].city, 'São Paulo/SP'); + assert.equal(seen.some(([, row]) => 'email' in row || 'address1' in row), false); + }); + + it('reassembles a row larger than the read buffer', async () => { + const body = JSON.parse(dumpText()); + body.net.data[1].notes = 'x'.repeat(CHUNK + CHUNK / 2); + const seen = await collect(writeDump('big-row.json', JSON.stringify(body))); + assert.equal(seen[1][1].notes.length, CHUNK + CHUNK / 2); + assert.equal(seen.length, 6); + }); + + it('reassembles a table key split across reads', async () => { + // Pad so the "netixlan" key's quote sits 4 bytes before the 1 MB boundary. + const unpadded = dumpText({ padBefore: 'p' }); + const keyAt = Buffer.byteLength(unpadded.slice(0, unpadded.indexOf('"netixlan"'))); + const text = dumpText({ padBefore: 'p'.repeat(1 + CHUNK - 4 - keyAt) }); + const keyByte = Buffer.byteLength(text.slice(0, text.indexOf('"netixlan"'))); + assert.ok(keyByte < CHUNK && keyByte + 10 > CHUNK, `key at ${keyByte}`); + const seen = await collect(writeDump('split-key.json', text)); + assert.equal(seen.filter(([table]) => table === 'netixlan').length, 2); + }); + + it('rejects a truncated dump', async () => { + const text = dumpText(); + await assert.rejects(collect(writeDump('cut.json', text.slice(0, text.length - 40))), /truncated/); + await assert.rejects(collect(writeDump('empty.json', '')), /truncated/); + }); + + it('stops when the signal aborts', async () => { + const file = writeDump('abort.json', dumpText()); + await assert.rejects(scanDumpRows(file, new Set(['net']), () => {}, { signal: AbortSignal.abort() })); + }); +}); + +describe('writeIndexFile', () => { + it('writes the same bytes as JSON.stringify, across batches', async () => { + const index = { v: 1, ix: { 1: ['A "x"', '', 'DE'] }, fac: {}, nets: {} }; + for (let asn = 1; asn <= 30000; asn++) index.nets[asn] = { policy: 'Open', irr: `AS-SET-${asn}`, ix: [[1, 1000, 1]] }; + const file = path.join(dir, 'written.json'); + await writeIndexFile(file, index); + assert.equal(fs.readFileSync(file, 'utf8'), JSON.stringify(index)); + }); +}); + +describe('distillPeeringdbDump', () => { + it('turns a dump into a loadable index without contact data', async () => { + const out = path.join(dir, 'index.json'); + const counts = await distillPeeringdbDump(writeDump('full.json', dumpText()), out); + assert.deepEqual(counts, { rows: 6, networks: 2, exchanges: 1, facilities: 1 }); + + const text = fs.readFileSync(out, 'utf8'); + assert.equal(text.includes('@'), false); + assert.equal(text.includes('555'), false); + assert.equal(text.includes('Main Street'), false); + + const raw = tables(); + const expected = buildPeeringdbIndex(Object.fromEntries( + Object.entries(raw).map(([table, { data }]) => [table, data.map(projectRow[table])]), + )); + assert.equal(text, JSON.stringify(expected)); + + const net = expandNet(readPeeringdbIndex(out), 64500); + assert.deepEqual(net.ixs, [{ name: 'IX "Quoted" São Paulo', city: 'São Paulo/SP', country: 'BR', speed: 20000, rsPeer: true }]); + assert.equal(net.ratio, 'Balanced'); + }); + + it('refuses a dump missing one of the four tables', async () => { + await assert.rejects( + distillPeeringdbDump(writeDump('no-netfac.json', dumpText({ omit: 'netfac' })), path.join(dir, 'x.json')), + /no netfac rows/, + ); + }); +});