Fix(datasets): watch from before the boot downloads; honour hand-named CAIDA files

- The watcher started only after every boot step, so a dataset published
  by another process (or placed by hand) while another step was running
  became the watcher's baseline and was never loaded. It now starts before
  the boot downloads.
- The CAIDA readers serve the newest *.txt under any name, but the boot
  download checked only the published names, so an upgrade with a
  hand-named snapshot re-downloaded it. A row can now supply hasSnapshot();
  as2org / as-rel use their reader's own rule.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend-server.js

@@ -367,6 +367,11 @@ const bootBackend = async () => {
         logger.info(`🚀 Backend server ready on http://localhost:${backEndPort}`);
     });
 
+    // Watching from before the downloads: a dataset another process
+    // publishes (or someone places by hand) while a boot step is still
+    // running is reloaded, not taken as the watcher's baseline.
+    watchDatasets(datasets);
+
     await runOfflineBootstrap([
         async () => {
             await bootstrapDatasets(datasets);
@@ -379,7 +384,6 @@ const bootBackend = async () => {
         bootstrapServiceStatus,
     ]);
 
-    watchDatasets(datasets);
     startDatasetScheduler(datasets);
     startServiceStatusPolling();
 };

common/as-org-db.js

@@ -97,6 +97,9 @@ export function reloadAsOrgDatabase(reason = 'reload') {
 /** Whether a snapshot is loaded (false until the updater has fetched one). */
 export const isAsOrgLoaded = () => loadedFrom !== null;
 
+/** Whether a snapshot file is on disk under any name this reader accepts. */
+export const hasAsOrgSnapshot = () => findSnapshot() !== null;
+
 /** Org name for an ASN, or null when the snapshot doesn't have it. */
 export function lookupAsOrgName(asn) {
     return asnToOrgName.get(Number(asn)) || null;

common/as-rel-db.js

@@ -162,6 +162,9 @@ export function customerCountOf(asn) {
 /** Whether a snapshot is loaded (false until the updater has fetched one). */
 export const isAsRelLoaded = () => loadedFrom !== null;
 
+/** Whether a snapshot file is on disk under any name this reader accepts. */
+export const hasAsRelSnapshot = () => findSnapshot() !== null;
+
 /** Whether this ASN is in the CAIDA-derived Tier 1 set. */
 export function isTier1(asn) {
     return tier1Set.has(Number(asn));

common/dataset-updater.js

@@ -26,6 +26,10 @@
 //                     name in `files`
 //   validate(staged)  throws to refuse a truncated or corrupt download
 //   reload(reason)    swaps the in-memory copy for the published files
+//   hasSnapshot()     optional; whether the reader already has a snapshot
+//                     to serve, when it accepts more than the published file
+//                     names (a hand-named CAIDA file) — the boot download
+//                     then skips. Default: every published file is present
 //   legacyState       optional { file, toState(json) → { identifier, updatedAt } }:
 //                     the state file an earlier updater wrote, read once so
 //                     an upgrade doesn't re-download
@@ -89,6 +93,9 @@ export const isRowEnabled = (row) => !row.enabled || Boolean(row.enabled());
 /** Whether every file the row publishes is on disk. */
 export const isPresent = (row) => row.files.every((file) => fs.existsSync(path.join(row.dir, file)));
 
+// Whether the boot download can skip the row: its reader has something.
+const hasSnapshot = (row) => (row.hasSnapshot ? row.hasSnapshot() : isPresent(row));
+
 /** The row's state, falling back to an earlier updater's state file; {} when neither exists. */
 export const readState = async (row) => {
     const read = async (file) => JSON.parse(await fsp.readFile(path.join(row.dir, file), 'utf8'));
@@ -262,7 +269,7 @@ export const updateDataset = async (row, { signal, reason = 'auto update', wait
  */
 export const bootstrapDataset = async (row, { timeoutMs = BOOTSTRAP_TIMEOUT_MS } = {}) => {
     if (!isRowEnabled(row)) return { status: 'disabled' };
-    if (isPresent(row)) return { status: 'present' };
+    if (hasSnapshot(row)) return { status: 'present' };
 
     const minutes = Math.round(timeoutMs / 60000);
     logger.warn(`📥 Dataset ${row.id} missing; downloading (timeout ${minutes} min)...`);

common/datasets.js

@@ -22,8 +22,8 @@ import maxmind from 'maxmind';
 import logger from './logger.js';
 import { fetchUpstream } from './fetch-with-timeout.js';
 import { downloadToFile, decompressFile } from './dataset-updater.js';
-import { AS_ORG_DB_DIR, AS_ORG_FILE, reloadAsOrgDatabase } from './as-org-db.js';
-import { AS_REL_DB_DIR, AS_REL_FILE, reloadAsRelDatabase } from './as-rel-db.js';
+import { AS_ORG_DB_DIR, AS_ORG_FILE, reloadAsOrgDatabase, hasAsOrgSnapshot } from './as-org-db.js';
+import { AS_REL_DB_DIR, AS_REL_FILE, reloadAsRelDatabase, hasAsRelSnapshot } from './as-rel-db.js';
 import { PEERINGDB_DB_DIR, PEERINGDB_FILE, readPeeringdbIndex, reloadPeeringdbDatabase } from './peeringdb-db.js';
 import { distillPeeringdbDump } from './peeringdb-distill.js';
 import { hasRadarApiKey } from './cf-radar.js';
@@ -201,6 +201,8 @@ export const datasets = [
         id: 'as2org',
         dir: AS_ORG_DB_DIR,
         files: [AS_ORG_FILE],
+        // The reader takes the newest *.txt, whatever its name.
+        hasSnapshot: hasAsOrgSnapshot,
         legacyState: CAIDA_LEGACY_STATE,
         legacyAutoUpdateEnv: 'CAIDA_AUTO_UPDATE',
         // Stable 'latest' symlink server-side → HEAD + Last-Modified is enough
@@ -219,6 +221,8 @@ export const datasets = [
         id: 'as-rel',
         dir: AS_REL_DB_DIR,
         files: [AS_REL_FILE],
+        // The reader takes the newest *.txt, whatever its name.
+        hasSnapshot: hasAsRelSnapshot,
         legacyState: CAIDA_LEGACY_STATE,
         legacyAutoUpdateEnv: 'CAIDA_AUTO_UPDATE',
         // No 'latest' symlink — scrape the directory listing; YYYYMMDD

tests/dataset-updater.test.js

@@ -225,6 +225,12 @@ describe('bootstrapDataset', () => {
         assert.equal(log.fetches, 1);
     });
 
+    it('trusts a row\'s own snapshot probe (a reader that accepts other file names)', async () => {
+        const { row, log } = makeRow({ hasSnapshot: () => true });
+        assert.equal((await bootstrapDataset(row)).status, 'present');
+        assert.equal(log.fetches, 0, 'the published names are absent, the reader has a snapshot');
+    });
+
     it('downloads a missing dataset and reloads it as a bootstrap', async () => {
         const { row, log } = makeRow();
         assert.equal((await bootstrapDataset(row)).status, 'downloaded');

tests/datasets.test.js

@@ -205,6 +205,12 @@ describe('CAIDA rows', () => {
         assert.deepEqual(requested, []);
     });
 
+    it('count a hand-named *.txt as a snapshot at boot, as their readers do', async () => {
+        for (const id of ['as2org', 'as-rel']) {
+            assert.equal(typeof row(id).hasSnapshot, 'function', id);
+        }
+    });
+
     it('keep the pre-engine CAIDA_AUTO_UPDATE as their legacy schedule flag', () => {
         for (const id of ['as2org', 'as-rel', 'peeringdb']) {
             assert.equal(row(id).legacyAutoUpdateEnv, 'CAIDA_AUTO_UPDATE', id);