autonomous/vector_memory.mjs

/**
 * BOB Vector Memory — pgvector on Neon Postgres
 *
 * Semantic anchors for every knowledge chunk. When BOB needs to "speak",
 * he performs a similarity search here — the result is the "linguistic vibe"
 * that constrains how the theorem gets transcoded into natural speech.
 *
 * Phase 1: Neon Postgres + pgvector, deterministic 64-dim vectors
 * Phase 2: Replace deterministic vectors with Granite embeddings (768-dim)
 *
 * The pgvector similarity search replaces keyword lookup with:
 * "What is the SEMANTIC NEIGHBORHOOD of this theorem?" → oracle lens guide
 */

import { REGISTRY, chunkVector, cosine } from './unicode_chunks.mjs'
import { THEOREMS, getTheorem }          from './lisp_theorems.mjs'

// ── Neon connection ───────────────────────────────────────────────────────────

let pg = null   // lazy-loaded postgres client

async function getDB(dbUrl) {
  if (pg) return pg
  try {
    // Use postgres.js (pure JS, works in Node without native pg)
    const { default: Postgres } = await import('https://esm.sh/postgres@3')
    pg = Postgres(dbUrl, { ssl: 'require', max: 3 })
    return pg
  } catch {
    try {
      // Fallback: node-postgres
      const { default: Pg } = await import('pg')
      const client = new Pg.Client({ connectionString: dbUrl })
      await client.connect()
      pg = {
        async query(sql, params) { return client.query(sql, params) },
        end: () => client.end(),
      }
      return pg
    } catch { return null }
  }
}

// ── Schema bootstrap ──────────────────────────────────────────────────────────

const BOOTSTRAP_SQL = `
  CREATE EXTENSION IF NOT EXISTS vector;

  CREATE TABLE IF NOT EXISTS bob_vectors (
    id           SERIAL PRIMARY KEY,
    concept      VARCHAR(100) UNIQUE NOT NULL,
    theorem      TEXT NOT NULL,
    unicode_char TEXT,
    oracle_word  VARCHAR(50),
    domain       VARCHAR(50),
    vector       vector(64) NOT NULL,
    created_at   TIMESTAMPTZ DEFAULT NOW()
  );

  CREATE INDEX IF NOT EXISTS bob_vectors_cosine
    ON bob_vectors USING ivfflat (vector vector_cosine_ops)
    WITH (lists = 10);
`

// ── Memory operations ─────────────────────────────────────────────────────────

export async function initVectorMemory(dbUrl) {
  const db = await getDB(dbUrl)
  if (!db) { console.error('[vector_memory] DB unavailable'); return false }

  try {
    // Bootstrap schema
    for (const stmt of BOOTSTRAP_SQL.split(';').map(s => s.trim()).filter(Boolean)) {
      await db.query(stmt)
    }

    // Upsert all known chunks
    let count = 0
    for (const chunk of REGISTRY) {
      if (!chunk) continue
      const vec  = chunkVector(chunk.name)
      const theo = getTheorem(chunk.name) || `(${chunk.name.toUpperCase()} (IMPLIES (HAS-DOMAIN X) (ORACLE-CONSTRAINED X '${chunk.oracle})))`
      if (!vec) continue

      await db.query(`
        INSERT INTO bob_vectors (concept, theorem, unicode_char, oracle_word, domain, vector)
        VALUES ($1, $2, $3, $4, $5, $6::vector)
        ON CONFLICT (concept) DO UPDATE SET
          theorem = EXCLUDED.theorem,
          vector  = EXCLUDED.vector
      `, [
        chunk.name,
        theo,
        String.fromCodePoint(0xE000 + REGISTRY.indexOf(chunk)),
        chunk.oracle,
        chunk.domain,
        JSON.stringify(Array.from(vec)),
      ])
      count++
    }

    console.log(`[vector_memory] ${count} chunks upserted to pgvector`)
    return true
  } catch (e) {
    console.error('[vector_memory] init error:', e.message)
    return false
  }
}

// Semantic search — find k nearest concepts by vector similarity
export async function semanticSearch(concept, k = 3, dbUrl) {
  const db = await getDB(dbUrl)
  if (!db) return inMemorySearch(concept, k)

  const vec = chunkVector(concept)
  if (!vec) return inMemorySearch(concept, k)

  try {
    const result = await db.query(`
      SELECT concept, theorem, oracle_word, domain,
             1 - (vector <=> $1::vector) AS similarity
      FROM   bob_vectors
      WHERE  concept != $2
      ORDER  BY vector <=> $1::vector
      LIMIT  $3
    `, [JSON.stringify(Array.from(vec)), concept, k])

    return result.rows || []
  } catch {
    return inMemorySearch(concept, k)
  }
}

// In-memory fallback (no DB)
function inMemorySearch(concept, k = 3) {
  const qVec = chunkVector(concept)
  if (!qVec) return []
  return REGISTRY
    .filter(c => c && c.name.toLowerCase() !== concept.toLowerCase())
    .map(c => {
      const v = chunkVector(c.name)
      return v ? { concept: c.name, oracle_word: c.oracle, domain: c.domain, similarity: cosine(qVec, v) } : null
    })
    .filter(Boolean)
    .sort((a, b) => b.similarity - a.similarity)
    .slice(0, k)
}

// Get a single concept's semantic neighbors + use as oracle lens context
export async function getSematicContext(concept, dbUrl) {
  const neighbors = await semanticSearch(concept, 3, dbUrl)
  if (!neighbors.length) return null

  return {
    concept,
    neighbors: neighbors.map(n => ({
      name:       n.concept,
      oracle:     n.oracle_word,
      domain:     n.domain,
      similarity: n.similarity,
    })),
    // The neighbor with highest similarity provides the "vibe guide"
    primaryGuide: neighbors[0],
  }
}