ImBenjiandClaude Opus 5
6f1d1eee2d
Archive ingestion had been dead since 2026-08-02 because nothing in the compose stack actually ran it. Everything downstream starved from there. - add ingest + enrichment services. server.js only starts the scheduler when DURIIN_RUN_SCHEDULER is not "false", and workers/index.js was not running at all, so articles never got event_id/content/has_embedding and the coordinator had nothing to lease. - pass an explicit origin from coordinatorWorker. it was never passed, so acceptProposal defaulted to 'live' and 464 historical backfill predictions were recorded as live. that also meant verifyEvidence got a null cutoff and skipped its date check entirely. - coarsen cohortKey to event families + horizon buckets. 201 free text event types produced 221 cohorts averaging 2.76 samples, so the n>=30 gate could never be reached and everything abstained for the wrong reason. - gate on cohort diversity, not just sample count. one ticker was roughly half of all resolved outcomes, so a pure count gate was measuring one company. unknown diversity abstains rather than passing. - resolve the admin archive db explicitly and probe it. it relied on a Dockerfile symlink, and without it better-sqlite3 quietly creates an empty file and serves a phantom archive. - clamp implausible future publication dates at ingest. - pin the db backend to sqlite by default. compose hardcoded postgres "true", which would have overridden the operator's own .env on the next redeploy and pointed everything at a stale snapshot. scripts/repair-autonomy-labels.js relabels the affected rows. it is dry run by default and has not been applied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WnNxwxfXSbeNtjvtz5gayb
62 lines
2.6 KiB
JavaScript
62 lines
2.6 KiB
JavaScript
// Guard against publication dates that sit in the future.
|
|
//
|
|
// pub_date_effective is what the autonomy coordinator uses to derive a
|
|
// prediction's information_cutoff (max pub_date_effective across an event's
|
|
// articles), so a single bogus feed date drags the cutoff forward and quietly
|
|
// breaks evidence-cutoff enforcement and outcome scoring. Production currently
|
|
// has exactly one such row, but one is enough to poison an event.
|
|
//
|
|
// Tolerance: 48 hours. It has to swallow the legitimate cases —
|
|
// * date only strings ("2026-08-29") are stored as midnight UTC, and a
|
|
// publisher in UTC+14 can legitimately stamp tomorrow's date,
|
|
// * feeds that emit local time without an offset, worst case ~14h ahead,
|
|
// * modest clock skew on the publisher's box.
|
|
// 48h covers all of that with room to spare while still catching anything
|
|
// genuinely wrong — the offending production row is about four months out.
|
|
const DEFAULT_TOLERANCE_MS = 48 * 60 * 60 * 1000;
|
|
|
|
function toleranceMs() {
|
|
const hours = Number(process.env.INGEST_FUTURE_PUB_DATE_HOURS);
|
|
if (Number.isFinite(hours) && hours > 0) return hours * 60 * 60 * 1000;
|
|
return DEFAULT_TOLERANCE_MS;
|
|
}
|
|
|
|
// Returns { ok, value, skewMs, toleranceMs }. `value` is null when the date is
|
|
// implausible so the caller can fall back to ingestion time. The article itself
|
|
// is never dropped for this — a bad date is not a bad article.
|
|
function checkPubDate(value, now = Date.now(), tolerance = toleranceMs()) {
|
|
if (!value) return { ok: true, value: null, skewMs: 0, toleranceMs: tolerance };
|
|
|
|
const parsed = new Date(value).getTime();
|
|
if (Number.isNaN(parsed)) return { ok: true, value: null, skewMs: 0, toleranceMs: tolerance };
|
|
|
|
const skewMs = parsed - now;
|
|
if (skewMs > tolerance) {
|
|
return { ok: false, value: null, skewMs, toleranceMs: tolerance };
|
|
}
|
|
|
|
return { ok: true, value, skewMs, toleranceMs: tolerance };
|
|
}
|
|
|
|
|
|
// Same check, but it also does the shouting. Keeps ingest.js readable and makes
|
|
// sure every clamp lands in the logs with the source and the offending value.
|
|
function guardEffectivePubDate(pubDate, fallback, context = {}) {
|
|
const verdict = checkPubDate(pubDate);
|
|
if (verdict.ok) return pubDate || fallback;
|
|
|
|
const days = (verdict.skewMs / 86400000).toFixed(1);
|
|
console.warn(
|
|
`[ingest] refusing future pub date from "${context.source || 'unknown source'}": ${pubDate} is ${days} days ahead ` +
|
|
`(tolerance ${Math.round(verdict.toleranceMs / 3600000)}h) — pub_date_effective falls back to ${fallback}. url=${context.url || 'n/a'}`
|
|
);
|
|
|
|
return fallback;
|
|
}
|
|
|
|
module.exports = {
|
|
checkPubDate,
|
|
guardEffectivePubDate,
|
|
DEFAULT_TOLERANCE_MS,
|
|
};
|