ImBenjiandClaude Opus 5
6f1d1eee2d
Archive ingestion had been dead since 2026-08-02 because nothing in the compose stack actually ran it. Everything downstream starved from there. - add ingest + enrichment services. server.js only starts the scheduler when DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at all, so articles never got event_id/content/has_embedding and the coordinator had nothing to lease. - pass an explicit origin from coordinatorWorker. it was never passed, so acceptProposal defaulted to 'live' and 464 historical backfill predictions were recorded as live. that also meant verifyEvidence got a null cutoff and skipped its date check entirely. - coarsen cohortKey to event families + horizon buckets. 201 free text event types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could never be reached and everything abstained for the wrong reason. - gate on cohort diversity, not just sample count. one ticker was roughly half of all resolved outcomes, so a pure count gate was measuring one company. unknown diversity abstains rather than passing. - resolve the admin archive db explicitly and probe it. it relied on a Dockerfile symlink, and without it better-sqlite3 quietly creates an empty file and serves a phantom archive. - clamp implausible future publication dates at ingest. - pin the db backend to sqlite by default. compose hardcoded postgres "true", which would have overridden the operator's own .env on the next redeploy and pointed everything at a stale snapshot. scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by default and has not been applied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
75 lines
2.9 KiB
JavaScript
75 lines
2.9 KiB
JavaScript
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
|
|
const { checkPubDate, guardEffectivePubDate, DEFAULT_TOLERANCE_MS } = require('../src/pubDateGuard');
|
|
|
|
const NOW = Date.parse('2026-08-29T12:00:00.000Z');
|
|
const HOUR = 60 * 60 * 1000;
|
|
|
|
test('ordinary past publication dates pass straight through', () => {
|
|
const verdict = checkPubDate('2026-08-27T09:30:00.000Z', NOW);
|
|
assert.equal(verdict.ok, true);
|
|
assert.equal(verdict.value, '2026-08-27T09:30:00.000Z');
|
|
});
|
|
|
|
test('a date-only feed value from an eastern timezone is still accepted', () => {
|
|
// "2026-08-30" stored as midnight UTC is 12 hours ahead of now — legitimate
|
|
const verdict = checkPubDate('2026-08-30T00:00:00.000Z', NOW);
|
|
assert.equal(verdict.ok, true);
|
|
});
|
|
|
|
test('mild clock skew inside the tolerance is accepted', () => {
|
|
const verdict = checkPubDate(new Date(NOW + 47 * HOUR).toISOString(), NOW);
|
|
assert.equal(verdict.ok, true);
|
|
});
|
|
|
|
test('anything past the tolerance is rejected', () => {
|
|
const verdict = checkPubDate(new Date(NOW + 49 * HOUR).toISOString(), NOW);
|
|
assert.equal(verdict.ok, false);
|
|
assert.equal(verdict.value, null);
|
|
assert.ok(verdict.skewMs > DEFAULT_TOLERANCE_MS);
|
|
});
|
|
|
|
test('the real production offender is caught', () => {
|
|
const verdict = checkPubDate('2026-12-22T00:00:00.000Z', NOW);
|
|
assert.equal(verdict.ok, false);
|
|
});
|
|
|
|
test('missing and unparseable dates are not treated as future dates', () => {
|
|
assert.equal(checkPubDate(null, NOW).ok, true);
|
|
assert.equal(checkPubDate('', NOW).ok, true);
|
|
assert.equal(checkPubDate('not a date at all', NOW).ok, true);
|
|
assert.equal(checkPubDate('not a date at all', NOW).value, null);
|
|
});
|
|
|
|
test('the tolerance boundary itself is inclusive', () => {
|
|
assert.equal(checkPubDate(new Date(NOW + DEFAULT_TOLERANCE_MS).toISOString(), NOW).ok, true);
|
|
assert.equal(checkPubDate(new Date(NOW + DEFAULT_TOLERANCE_MS + 1).toISOString(), NOW).ok, false);
|
|
});
|
|
|
|
test('a rejected date falls back to ingestion time and never drops the article', () => {
|
|
const ingestedAt = new Date().toISOString();
|
|
const future = new Date(Date.now() + 120 * 24 * HOUR).toISOString();
|
|
|
|
const warnings = [];
|
|
const original = console.warn;
|
|
console.warn = (message) => warnings.push(message);
|
|
try {
|
|
const effective = guardEffectivePubDate(future, ingestedAt, { source: 'gdelt', url: 'https://example.com/a' });
|
|
assert.equal(effective, ingestedAt);
|
|
} finally {
|
|
console.warn = original;
|
|
}
|
|
|
|
assert.equal(warnings.length, 1);
|
|
assert.match(warnings[0], /gdelt/);
|
|
assert.match(warnings[0], /https:\/\/example\.com\/a/);
|
|
assert.ok(warnings[0].includes(future));
|
|
});
|
|
|
|
test('a good date is kept, and a missing one falls back quietly', () => {
|
|
const ingestedAt = '2026-08-29T12:00:00.000Z';
|
|
assert.equal(guardEffectivePubDate('2026-08-01T00:00:00.000Z', ingestedAt, {}), '2026-08-01T00:00:00.000Z');
|
|
assert.equal(guardEffectivePubDate(null, ingestedAt, {}), ingestedAt);
|
|
});
|