Duriin-API/src/db/postgres.js

// postgres adapter that mimics the better-sqlite3 synchronous interface.
// uses deasync to block the event loop until queries resolve so all
// existing consumers (which call .get()/.all()/.run() synchronously) keep working.

const { Pool } = require('pg');
const deasync = require('deasync');
const config = require('../config');

const pool = new Pool(config.database.postgres);

function querySync(sql, params) {
  let done = false;
  let result, err;

  pool.query(sql, params).then(r => { result = r; done = true; }).catch(e => { err = e; done = true; });
  deasync.loopWhile(() => !done);

  if (err) throw err;
  return result;
}

// translate ? placeholders to $1, $2, ... for postgres
function toPositional(sql) {
  let i = 0;
  return sql.replace(/\?/g, () => `$${++i}`);
}


// sqlite-vec uses:  WHERE embedding MATCH ? AND k = ?
// pgvector uses:    ORDER BY embedding <-> $1::vector LIMIT $2
const NEAREST_NEIGHBORS_RE = /SELECT\s+article_id\s*,\s*distance\s+FROM\s+article_embeddings\s+WHERE\s+embedding\s+MATCH\s+\?\s+AND\s+k\s*=\s*\?\s+ORDER\s+BY\s+distance/is;

function rewriteNearestNeighbors(sql) {
  if (!NEAREST_NEIGHBORS_RE.test(sql)) return null;
  return `
    SELECT article_id, (embedding <-> $1::vector) AS distance
    FROM article_embeddings
    ORDER BY embedding <-> $1::vector
    LIMIT $2
  `;
}

function Statement(sql) {
  const pgSql = rewriteNearestNeighbors(sql) || toPositional(sql);

  return {
    get(...params) {
      const r = querySync(pgSql, params.flat());
      return r.rows[0] ?? undefined;
    },

    all(...params) {
      const r = querySync(pgSql, params.flat());
      return r.rows;
    },

    run(...params) {
      const r = querySync(pgSql, params.flat());
      return { changes: r.rowCount, lastInsertRowid: null };
    },
  };
}

function exec(sql) {
  // split on ; and run each statement individually (DDL blocks)
  const stmts = sql.split(';').map(s => s.trim()).filter(Boolean);
  for (const stmt of stmts) {
    querySync(stmt, []);
  }
}

function transaction(fn) {
  return (...args) => {
    querySync('BEGIN', []);
    try {
      const result = fn(...args);
      querySync('COMMIT', []);
      return result;
    } catch (e) {
      querySync('ROLLBACK', []);
      throw e;
    }
  };
}

function prepare(sql) {
  return Statement(sql);
}

const pgDb = { prepare, exec, transaction, pool };

// create the schema in postgres, mirroring src/db.js

exec(`
  CREATE TABLE IF NOT EXISTS events (
    id SERIAL PRIMARY KEY,
    title TEXT NOT NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS articles (
    id SERIAL PRIMARY KEY,
    title TEXT NOT NULL,
    description TEXT,
    content TEXT,
    image TEXT,
    content_status TEXT,
    content_error TEXT,
    content_attempted_at TEXT,
    content_attempt_count INTEGER NOT NULL DEFAULT 0,
    content_retry_after TEXT,
    is_index_page INTEGER NOT NULL DEFAULT 0,
    has_embedding INTEGER NOT NULL DEFAULT 0,
    url TEXT NOT NULL UNIQUE,
    normalized_title TEXT NOT NULL,
    source TEXT NOT NULL,
    pub_date TEXT,
    pub_date_effective TEXT,
    language TEXT,
    event_id INTEGER REFERENCES events(id),
    ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
  )
`);

exec(`CREATE INDEX IF NOT EXISTS idx_articles_source ON articles(source)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_pub_date ON articles(pub_date)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_ingested_at ON articles(ingested_at)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_normalized_title ON articles(normalized_title)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_event_id ON articles(event_id)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_has_embedding ON articles(has_embedding)`);
exec(`CREATE INDEX IF NOT EXISTS idx_articles_pub_date_effective ON articles(pub_date_effective DESC)`);

exec(`
  CREATE TABLE IF NOT EXISTS article_embedding_store (
    article_id INTEGER NOT NULL,
    model TEXT NOT NULL,
    embedding BYTEA NOT NULL,
    embedded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (article_id, model)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS article_embedding_meta (
    article_id INTEGER PRIMARY KEY,
    model TEXT NOT NULL,
    embedded_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS article_embeddings (
    article_id INTEGER PRIMARY KEY,
    embedding vector(8192)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS query_embeddings (
    query TEXT NOT NULL,
    model TEXT NOT NULL,
    embedding BYTEA NOT NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (query, model)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS gdelt_backfill_windows (
    source_id TEXT NOT NULL,
    window_start TEXT NOT NULL,
    window_end TEXT NOT NULL,
    completed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (source_id, window_start, window_end)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS crawler_page_classifications (
    url TEXT PRIMARY KEY,
    site_name TEXT NOT NULL,
    classification TEXT NOT NULL,
    confidence DOUBLE PRECISION,
    pattern TEXT,
    classified_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS crawler_url_patterns (
    site_name TEXT NOT NULL,
    pattern TEXT NOT NULL,
    classification TEXT NOT NULL,
    hit_count INTEGER NOT NULL DEFAULT 1,
    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (site_name, pattern)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS crawler_site_rules (
    site_name TEXT NOT NULL,
    rule_type TEXT NOT NULL,
    rule_value TEXT NOT NULL,
    classification TEXT NOT NULL,
    hit_count INTEGER NOT NULL DEFAULT 1,
    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (site_name, rule_type, rule_value)
  )
`);

exec(`
  CREATE TABLE IF NOT EXISTS domain_fetch_policy (
    domain TEXT PRIMARY KEY,
    policy TEXT NOT NULL DEFAULT 'auto',
    consecutive_plain_failures INTEGER NOT NULL DEFAULT 0,
    consecutive_browser_failures INTEGER NOT NULL DEFAULT 0,
    plain_success_count INTEGER NOT NULL DEFAULT 0,
    browser_success_count INTEGER NOT NULL DEFAULT 0,
    expires_at TEXT,
    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
  )
`);

// same story as the sqlite side: the creates above skip a database that already
// has the table, so additions land here too. postgres does have the IF NOT EXISTS
// form so this one stays boring. Additive only.
exec(`
  ALTER TABLE crawler_page_classifications
  ADD COLUMN IF NOT EXISTS confidence DOUBLE PRECISION
`);

module.exports = pgDb;