RAG++ SOTA Improvements

┌─────────────────────────────────────────────────────────────────────┐
│                         RAG++ Core                                   │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐               │
│  │   distance/  │  │ quantization/│  │    sparse/   │               │
│  │              │  │              │  │              │               │
│  │  • scalar    │  │  • sq8       │  │  • bm25      │               │
│  │  • simd_avx2 │  │  • pq        │  │  • hybrid    │               │
│  │              │  │              │  │  • tokenizer │               │
│  └──────┬───────┘  └──────┬───────┘  └──────┬───────┘               │
│         │                 │                 │                        │
│         └────────────┬────┴────────────────┬┘                        │
│                      │                     │                         │
│  ┌───────────────────▼─────────────────────▼───────────────────────┐│
│  │                        index/                                    ││
│  │                                                                  ││
│  │  • FlatIndex        • HNSWIndex        • ParallelHNSWIndex      ││
│  │  • IndexRegistry    • ParallelHNSWBuilder                       ││
│  └──────────────────────────┬───────────────────────────────────────┘│
│                             │                                        │
│  ┌──────────────────────────▼───────────────────────────────────────┐│
│  │                      retrieval/                                  ││
│  │                                                                  ││
│  │  • QueryEngine           • HybridQueryEngine                     ││
│  │  • Reranker              • HybridSearcher                        ││
│  └──────────────────────────────────────────────────────────────────┘│
└─────────────────────────────────────────────────────────────────────┘

use rag_plusplus_core::{l2_distance_fast, inner_product_fast, cosine_similarity_fast};

// Automatic dispatch - uses AVX2 if available, falls back to scalar
let dist = l2_distance_fast(&vec_a, &vec_b);
let ip = inner_product_fast(&vec_a, &vec_b);
let cos = cosine_similarity_fast(&vec_a, &vec_b);

// Or use compute_distance for index-compatible interface
use rag_plusplus_core::{compute_distance, DistanceType};
let dist = compute_distance(&vec_a, &vec_b, DistanceType::L2);

// Check if AVX2 is available
#[cfg(target_arch = "x86_64")]
fn has_avx2() -> bool {
    is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma")
}

For each dimension d:
  1. Compute min_d, max_d from training data
  2. scale_d = 255.0 / (max_d - min_d)
  3. Encode: q[d] = round((x[d] - min_d) * scale_d)
  4. Decode: x'[d] = q[d] / scale_d + min_d

use rag_plusplus_core::{SQ8Quantizer, Quantizer, AsymmetricDistance};

// Train on representative vectors
let training_data: Vec<Vec<f32>> = load_training_vectors();
let quantizer = SQ8Quantizer::train(&training_data);

// Encode corpus
let encoded: Vec<SQ8EncodedVector> = corpus
    .iter()
    .map(|v| quantizer.encode(v))
    .collect();

// Asymmetric distance (full precision query vs quantized corpus)
let query: Vec<f32> = get_query();
for encoded_vec in &encoded {
    let dist = quantizer.asymmetric_l2(&query, encoded_vec);
    // dist is approximate L2 distance
}

Training:
  1. Split D-dim vectors into M subvectors of D/M dimensions
  2. For each subspace m:
     - Run k-means with K centroids
     - Store codebook[m] = K centroids

Encoding:
  For each subvector m:
    code[m] = argmin_k ||x[m] - codebook[m][k]||

Distance (ADC - Asymmetric Distance Computation):
  1. Precompute distance table: table[m][k] = ||q[m] - codebook[m][k]||²
  2. For each encoded vector: dist = Σ table[m][code[m]]

use rag_plusplus_core::{PQQuantizer, PQConfig, Quantizer};

// Configure PQ
let config = PQConfig {
    m: 16,           // Number of subspaces (compression = dim * 4 / m)
    k: 256,          // Centroids per subspace (256 = 1 byte per code)
    kmeans_iters: 25,
    seed: Some(42),
};

// Train
let quantizer = PQQuantizer::train_with_config(&training_data, &config);

// Encode corpus
let encoded: Vec<PQEncodedVector> = corpus
    .iter()
    .map(|v| quantizer.encode(v))
    .collect();

// Search with precomputed distance table (fast!)
let query: Vec<f32> = get_query();
let table = quantizer.compute_distance_table(&query);

for enc in &encoded {
    // O(M) distance computation instead of O(D)
    let dist_sq = quantizer.asymmetric_l2_squared_with_table(&table, enc);
}

1. Insert first node (entry point) sequentially
2. Partition remaining nodes into batches
3. For each batch in parallel:
   - Search for insertion point (read locks)
   - Update graph edges (write locks on affected nodes)
4. Maintain entry point atomically

use rag_plusplus_core::{ParallelHNSWBuilder, HNSWConfig};

let config = HNSWConfig::new(128)  // dimension
    .with_m(16)                     // max connections per layer
    .with_ef_construction(100);     // search width during build

let vectors: Vec<(String, Vec<f32>)> = load_vectors();

// Build with automatic thread count
let index = ParallelHNSWBuilder::new()
    .with_config(config)
    .with_seed(42)  // reproducible builds
    .build(vectors)?;

// Or specify thread count
let index = ParallelHNSWBuilder::new()
    .with_config(config)
    .with_threads(8)
    .with_batch_size(1000)
    .build(vectors)?;

// Search as normal
let results = index.search(&query, 10)?;

use rag_plusplus_core::{BM25Index, BM25Config};

let mut index = BM25Index::new();

// Or with custom config
let config = BM25Config {
    k1: 1.2,      // Term frequency saturation
    b: 0.75,      // Length normalization
    min_idf: 0.0, // IDF threshold
};
let mut index = BM25Index::with_config(config);

// Add documents
index.add("doc-1".into(), "machine learning neural networks");
index.add("doc-2".into(), "deep learning transformers attention");

// Search
let results = index.search("neural network architecture", 10);

use rag_plusplus_core::{HybridSearcher, HybridSearchConfig, HybridFusionStrategy};

let config = HybridSearchConfig {
    strategy: HybridFusionStrategy::RRF { k: 60.0 },
    candidates_per_index: 100,
    min_sparse_score: 0.0,
};

let searcher = HybridSearcher::new(&dense_index, &sparse_index)
    .with_config(config);

// Search with both vector and text
let results = searcher.search(
    &query_embedding,
    "transformer attention mechanism",
    10
)?;

// Results include both score components
for r in results {
    println!("{}: fused={:.3}, dense={:?}, sparse={:?}",
        r.id, r.score, r.dense_score, r.sparse_score);
}

use rag_plusplus_core::{
    HybridQueryEngine, HybridQueryEngineConfig, HybridQueryRequest,
    HybridFusionStrategy,
};

// Configure
let config = HybridQueryEngineConfig::new()
    .with_default_k(10)
    .with_fusion_strategy(HybridFusionStrategy::RRF { k: 60.0 })
    .with_candidates_per_index(100);

// Create engine
let engine = HybridQueryEngine::new(config, &dense_index, &sparse_index, &store);

// Query
let request = HybridQueryRequest::new(embedding)
    .with_text("neural network training")
    .with_k(10)
    .with_filter(FilterExpr::Eq("category".into(), "ml".into()));

let response = engine.query(request)?;

// Response includes
println!("Results: {}", response.len());
println!("Latency: {:?}", response.latency);
println!("Used hybrid: {}", response.used_hybrid);

if let Some(priors) = response.priors {
    println!("Prior mean: {}", priors.mean_outcome);
    println!("Prior CI: {:?}", priors.confidence_interval);
}

# All benchmarks
cargo bench

# Specific benchmark
cargo bench --bench e2e_benchmark
cargo bench --bench quantization
cargo bench --bench distance

// Quantization
pub use quantization::{
    Quantizer,              // Trait for all quantizers
    AsymmetricDistance,     // Trait for ADC
    SQ8Quantizer,           // Scalar quantization
    SQ8EncodedVector,
    PQQuantizer,            // Product quantization
    PQEncodedVector,
    PQDistanceTable,
    PQConfig,
};

// Sparse Retrieval
pub use sparse::{
    BM25Index,              // BM25 inverted index
    BM25Config,
    SparseResult,
    SimpleTokenizer,        // Default tokenizer
    Tokenizer,              // Tokenizer trait
    HybridSearcher,         // Dense + Sparse fusion
    HybridSearchConfig,
    HybridFusionStrategy,
    HybridResult,
};

// Parallel Index
pub use index::{
    ParallelHNSWBuilder,    // Parallel construction
    ParallelHNSWIndex,      // Thread-safe HNSW
};

// Hybrid Query Engine
pub use retrieval::{
    HybridQueryEngine,
    HybridQueryEngineConfig,
    HybridQueryRequest,
    HybridQueryResponse,
    HybridRetrievedRecord,
};

// Distance (SIMD-accelerated)
pub use distance::{
    l2_distance_fast,
    inner_product_fast,
    cosine_similarity_fast,
    compute_distance,       // Unified interface
};

// Before (still works)
use rag_plusplus_core::l2_distance;

// After (automatic SIMD when available)
use rag_plusplus_core::l2_distance_fast;

// Add quantized corpus alongside existing
let sq8 = SQ8Quantizer::train(&sample_vectors);
let quantized_corpus: Vec<_> = corpus.iter().map(|v| sq8.encode(v)).collect();

// Create BM25 index alongside dense index
let mut bm25 = BM25Index::new();
for (id, text) in documents {
    bm25.add(id, &text);
}

// Use HybridSearcher instead of direct index search
let searcher = HybridSearcher::new(&dense_index, &bm25);

// Before
let mut index = HNSWIndex::new(config);
for (id, vec) in vectors {
    index.add(id, &vec)?;
}

// After (parallel)
let index = ParallelHNSWBuilder::new()
    .with_config(config)
    .build(vectors)?;