Stage 3: Expand + Master Plan -- AI-Generated Graph Topology

-- Migration: v1.1.0 -- Confidence Evolution
-- Run on cloud-vm PostgreSQL

-- Step 1: Add updated_at column
ALTER TABLE knowledge_graph
  ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();

-- Backfill: set updated_at = created_at for existing rows
UPDATE knowledge_graph SET updated_at = created_at WHERE updated_at IS NULL;

-- Step 2: Add soft delete columns
ALTER TABLE knowledge_graph
  ADD COLUMN IF NOT EXISTS pruned_at TIMESTAMPTZ DEFAULT NULL;
ALTER TABLE knowledge_graph
  ADD COLUMN IF NOT EXISTS pruned_reason TEXT DEFAULT NULL;

-- Step 3: Partial index for active (non-pruned) triples
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_knowledge_active
  ON knowledge_graph (subject, predicate, object)
  WHERE pruned_at IS NULL;

-- Step 4: Index for decay queries (find stale triples)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_knowledge_updated
  ON knowledge_graph (updated_at)
  WHERE pruned_at IS NULL;

-- Step 5: Predicate registry table
CREATE TABLE IF NOT EXISTS predicate_registry (
  name TEXT PRIMARY KEY,
  tier INTEGER NOT NULL DEFAULT 3,  -- 1=Core, 2=Semi, 3=Freeform
  usage_count BIGINT NOT NULL DEFAULT 0,
  avg_confidence FLOAT NOT NULL DEFAULT 0.5,
  std_confidence FLOAT NOT NULL DEFAULT 0.0,
  information_content FLOAT NOT NULL DEFAULT 0.0,
  computed_threshold FLOAT NOT NULL DEFAULT 0.3,
  promoted_at TIMESTAMPTZ,
  created_at TIMESTAMPTZ DEFAULT NOW(),
  updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Step 6: Update upsert to touch updated_at
-- This is handled in Rust code change, not SQL.
-- The ON CONFLICT clause becomes:
-- ON CONFLICT (subject, predicate, object) DO UPDATE SET
--   confidence = GREATEST(knowledge_graph.confidence, EXCLUDED.confidence),
--   source = EXCLUDED.source,
--   updated_at = NOW()

// File: src/decay.rs (~100 lines)

use chrono::{DateTime, Utc, Duration};

pub struct DecayConfig {
    pub half_life_days: f64,      // Default: 90.0
    pub min_confidence: f64,       // Default: 0.05 (below this = prune candidate)
    pub exempt_predicates: Vec<String>,  // Tier 1 predicates, no decay
    pub exempt_sources: Vec<String>,     // e.g., "manual", "human-curated"
}

impl Default for DecayConfig {
    fn default() -> Self {
        Self {
            half_life_days: 90.0,
            min_confidence: 0.05,
            exempt_predicates: vec![
                "is_a".into(), "part_of".into(), "evolved_from".into(),
                "works_on".into(), "built_with".into(), "deployed_on".into(),
                "depends_on".into(),
            ],
            exempt_sources: vec!["manual".into(), "human-curated".into()],
        }
    }
}

pub fn compute_decayed_confidence(
    current_confidence: f64,
    updated_at: DateTime<Utc>,
    now: DateTime<Utc>,
    config: &DecayConfig,
) -> f64 {
    let days_since_update = (now - updated_at).num_seconds() as f64 / 86400.0;
    if days_since_update <= 0.0 {
        return current_confidence;
    }
    let decay_factor = 0.5_f64.powf(days_since_update / config.half_life_days);
    let decayed = current_confidence * decay_factor;
    decayed.max(config.min_confidence)
}

// SQL for batch decay (called from endpoint):
// UPDATE knowledge_graph
// SET confidence = confidence * POWER(0.5, EXTRACT(EPOCH FROM (NOW() - updated_at)) / 86400.0 / $1),
//     source = 'decay:' || source
// WHERE pruned_at IS NULL
//   AND predicate NOT IN (SELECT name FROM predicate_registry WHERE tier = 1)
//   AND source NOT LIKE 'decay:%'
//   AND updated_at < NOW() - INTERVAL '$2 days'

// File: src/topology.rs (~200 lines)

pub struct EntityMetrics {
    pub entity: String,
    pub in_degree: usize,
    pub out_degree: usize,
    pub total_degree: usize,
    pub predicate_diversity: f64,  // Shannon entropy
    pub avg_confidence: f64,
    pub min_confidence: f64,
    pub staleness_days: f64,
    pub isolation_score: f64,  // 1.0 - (degree / max_degree)
}

pub struct PredicateMetrics {
    pub predicate: String,
    pub edge_count: usize,
    pub avg_confidence: f64,
    pub std_confidence: f64,
    pub unique_subjects: usize,
    pub unique_objects: usize,
    pub density: f64,  // edges / (subjects * objects)
    pub information_content: f64,  // -log2(edge_count / total_edges)
}

// SQL for entity metrics:
// SELECT subject as entity,
//   COUNT(*) FILTER (WHERE subject = e) as out_degree,
//   COUNT(*) FILTER (WHERE object = e) as in_degree,
//   COUNT(DISTINCT predicate) as predicate_count,
//   AVG(confidence) as avg_confidence,
//   MIN(confidence) as min_confidence,
//   EXTRACT(EPOCH FROM (NOW() - MAX(updated_at))) / 86400 as staleness_days
// FROM knowledge_graph
// WHERE pruned_at IS NULL AND (subject = $1 OR object = $1)
// GROUP BY entity

// SQL for predicate metrics:
// SELECT predicate,
//   COUNT(*) as edge_count,
//   AVG(confidence) as avg_confidence,
//   STDDEV(confidence) as std_confidence,
//   COUNT(DISTINCT subject) as unique_subjects,
//   COUNT(DISTINCT object) as unique_objects,
//   -LOG(2, COUNT(*)::float / (SELECT COUNT(*) FROM knowledge_graph WHERE pruned_at IS NULL)) as info_content
// FROM knowledge_graph
// WHERE pruned_at IS NULL
// GROUP BY predicate
// ORDER BY edge_count DESC

# File: flows/feed-hub/gk_entity_embeddings.py (~180 lines)
# Prefect flow, runs weekly on cloud-vm

import httpx
import pickle
from pathlib import Path

GK_URL = "http://localhost:8001"
RAG_EMBED_URL = "http://localhost:8000/api/rag/embed"
OUTPUT_PATH = Path("/home/mohameddiomande/gk_entity_embeddings.pkl")

EXCLUDE_PREFIXES = ["session:", "prompt:", "agent:"]
NOISE_PREDICATES = {"contains_prompt", "has_intent", "ran_session", "has_path"}

def fetch_embeddable_entities():
    """Fetch entities with >= 2 non-noise predicates."""
    stats = httpx.get(f"{GK_URL}/api/knowledge/stats").json()
    total = stats["total_triples"]

    entities = {}  # entity -> list of (predicate, object, confidence)
    offset = 0
    while offset < total:
        resp = httpx.get(f"{GK_URL}/api/knowledge", params={"limit": 500}).json()
        for t in resp["triples"]:
            if t["predicate"] not in NOISE_PREDICATES:
                entities.setdefault(t["subject"], []).append(
                    (t["predicate"], t["object"], t["confidence"])
                )
        offset += 500
        if len(resp["triples"]) < 500:
            break

    # Filter: >= 2 non-noise predicates, no excluded prefixes
    return {
        e: preds for e, preds in entities.items()
        if len(preds) >= 2 and not any(e.startswith(p) for p in EXCLUDE_PREFIXES)
    }

def enrich_entity_text(entity, predicates):
    """Build context-rich text for embedding."""
    parts = [entity]
    for pred, obj, conf in sorted(predicates, key=lambda x: -x[2])[:10]:
        parts.append(f"{pred}: {obj} ({conf:.1f})")
    return ". ".join(parts)[:4000]

def embed_batch(texts, batch_size=50):
    """Embed texts via RAG++ endpoint."""
    embeddings = {}
    for i in range(0, len(texts), batch_size):
        batch = list(texts.items())[i:i+batch_size]
        for entity, text in batch:
            resp = httpx.post(RAG_EMBED_URL, json={"text": text}, timeout=10)
            if resp.status_code == 200:
                emb = resp.json().get("embedding", [])
                if len(emb) == 3072:
                    embeddings[entity] = emb
    return embeddings

# File: flows/feed-hub/gk_hierarchy_discovery.py (~200 lines)
# Runs after gk_entity_embeddings.py

import numpy as np
from scipy.cluster.hierarchy import fcluster, linkage
from scipy.spatial.distance import pdist
import httpx

def discover_hierarchy(embeddings, n_coarse=30, n_fine_per_coarse=5):
    """Two-level agglomerative clustering."""
    entities = list(embeddings.keys())
    vectors = np.array([embeddings[e] for e in entities])

    # Level 1: Coarse clustering
    linkage_matrix = linkage(pdist(vectors, metric='cosine'), method='ward')
    coarse_labels = fcluster(linkage_matrix, t=n_coarse, criterion='maxclust')

    # Level 2: Fine clustering within each coarse cluster
    hierarchy = {}  # {coarse_label: {fine_label: [entities]}}
    for cluster_id in range(1, n_coarse + 1):
        mask = coarse_labels == cluster_id
        cluster_entities = [entities[i] for i, m in enumerate(mask) if m]
        cluster_vectors = vectors[mask]

        if len(cluster_entities) <= n_fine_per_coarse:
            hierarchy[cluster_id] = {0: cluster_entities}
            continue

        fine_linkage = linkage(pdist(cluster_vectors, metric='cosine'), method='ward')
        n_fine = min(n_fine_per_coarse, len(cluster_entities) // 2)
        fine_labels = fcluster(fine_linkage, t=max(2, n_fine), criterion='maxclust')

        fine_clusters = {}
        for entity, label in zip(cluster_entities, fine_labels):
            fine_clusters.setdefault(label, []).append(entity)
        hierarchy[cluster_id] = fine_clusters

    return hierarchy

def label_clusters(hierarchy, embeddings):
    """Label clusters using shared predicates + Gemini fallback."""
    labels = {}
    for coarse_id, fine_clusters in hierarchy.items():
        coarse_entities = [e for fc in fine_clusters.values() for e in fc]
        coarse_label = auto_label(coarse_entities) or gemini_label(coarse_entities)
        labels[coarse_id] = coarse_label

        for fine_id, entities in fine_clusters.items():
            fine_label = auto_label(entities) or gemini_label(entities)
            labels[(coarse_id, fine_id)] = fine_label

    return labels

def auto_label(entities):
    """Try to label from shared GK predicates."""
    # Fetch predicates for all entities
    shared = None
    for entity in entities[:10]:
        resp = httpx.get(f"{GK_URL}/api/knowledge",
                        params={"subject": entity, "limit": 50}).json()
        preds = {(t["predicate"], t["object"]) for t in resp["triples"]}
        shared = preds if shared is None else shared & preds

    if shared:
        # Most common shared predicate-object pair
        return max(shared, key=lambda x: x[1])[1]  # Use the object as label
    return None

Wave 0.1 (migration) ─────> Wave 0.2 (upsert) ─────> Wave 0.3 (backfill)
                                                            |
                                                      Wave 0.4 (filter)
                                                            |
                                              ┌─────────────┴──────────────┐
                                              |                            |
                                        Wave 1.1-1.6               Wave 2.1-2.3
                                        (Confidence)                (Topology)
                                              |                            |
                                              |                      Wave 2.4-2.7
                                              |                      (Pruning)
                                              |                            |
                                              └────────────┬───────────────┘
                                                           |
                                                     Wave 3.1-3.5
                                                     (Registry)
                                                           |
                                                     Wave 3.5
                                                     (Deploy)
                                                           |
                                              ┌────────────┴───────────────┐
                                              |                            |
                                        Wave 4.1-4.7                Wave 5.1-5.2
                                        (Hierarchy)                  (KARL Loop)
                                              |                            |
                                              └────────────┬───────────────┘
                                                           |
                                                     Wave 5.3-5.6
                                                     (Planner + Obs)