Voice Ordering System - Technical Documentation

┌──────────────────────────────────────────────────────┐
│              iOS 26 SpeechAnalyzer                   │
│         (On-device speech-to-text)                   │
└─────────────────────┬────────────────────────────────┘
                      │ transcript
                      ▼
┌──────────────────────────────────────────────────────┐
│            EnhancedVoiceNLUEngine                    │
│  ┌─────────────────────────────────────────────┐     │
│  │ 1. MenuEmbeddingService (semantic search)   │     │
│  │ 2. SlotClassifier (extract size, milk, etc) │     │
│  │ 3. ClarificationPolicy (confidence checks)  │     │
│  │ 4. YAMLConstraintEngine (validate rules)    │     │
│  └─────────────────────────────────────────────┘     │
└─────────────────────┬────────────────────────────────┘
                      │ NLUResult
                      ▼
┌──────────────────────────────────────────────────────┐
│            VoiceDialogueManager                      │
│    (State machine: listen → process → confirm)       │
└──────────────────────────────────────────────────────┘

// Initialize and load embeddings
await embeddingService.loadAndEmbed(menuItems: items)

// Search for menu items
let results = await embeddingService.hybridSearch(query: "large iced vanilla latte", topK: 5)
// Returns: [(MenuSearchResult with item, score, matchType)]

let result = slotClassifier.extractSlots(from: "large iced oat milk latte", context: menuMatches)
// Returns: SlotExtractionResult with predictions and confidence scores

// User says: "I want a latte with... um... milk"
// System detects low confidence on milk type
// Response: "What kind of milk would you like? Whole, oat, almond, or soy?"

# Size constraints - what sizes are available
size_constraints:
  cappuccino:
    allowed: [small, medium]
    default: small
  latte:
    allowed: [small, medium, large, xl]
    default: medium

# Temperature constraints
temperature_constraints:
  cold_brew:
    allowed: [iced]
    default: iced

# Combination rules - invalid combinations
combination_rules:
  - name: no_iced_cappuccino
    item: cappuccino
    disallow:
      temperature: iced
    message: "Cappuccino is traditionally served hot"

# Seasonal items
seasonal_constraints:
  pumpkin_spice_latte:
    available: true
    start_month: 9
    end_month: 11

# Price modifiers
price_modifiers:
  alternative_milk:
    oat: 0.80
    almond: 0.70
  extra_espresso: 0.75

var order = VoiceParsedOrder(itemName: "cappuccino", size: .large)
let (processedOrder, result) = constraintEngine.processOrder(&order)

// If large cappuccino not allowed:
// - order.size automatically changed to .medium
// - result.autoFixes["size"] = "medium"
// - result.violations contains warning message

let engine = EnhancedVoiceNLUEngine()

// Initialize (loads embeddings - do this once at startup)
await engine.initialize()

// Process speech
let result = await engine.process(transcript: "large iced vanilla latte with oat milk")

// Result contains:
// - intent: .order
// - parsedOrders: [VoiceParsedOrder]
// - clarificationsNeeded: [ClarificationRequest]
// - overallConfidence: 0.85

Step 1: Normalize
├── Remove fillers: "can I get a large iced vanilla latte with oat milk"
├── Lowercase: "can i get a large iced vanilla latte with oat milk"
└── Output: "large iced vanilla latte oat milk"

Step 2: Intent Classification
├── Pattern match: "can i get" → order intent
└── Output: intent = .order, confidence = 0.95

Step 3: Menu Search (Semantic)
├── Embed query: [0.12, -0.34, 0.56, ...]
├── Search index: cosine similarity
└── Output: "Vanilla Latte" (score: 0.92)

Step 4: Slot Extraction
├── Size: "large" → large (confidence: 0.98)
├── Temperature: "iced" → iced (confidence: 0.97)
├── Milk: "oat milk" → oat (confidence: 0.95)
└── Syrup: "vanilla" → vanilla (confidence: 0.90)

Step 5: Clarification Check
├── All confidences > 0.6 ✓
├── No close alternatives ✓
└── Output: No clarification needed

Step 6: Constraint Validation
├── Vanilla Latte allows large ✓
├── Vanilla Latte allows iced ✓
├── Oat milk available ✓
└── Output: Valid order

Step 7: Build Result
└── VoiceParsedOrder(
      itemName: "Vanilla Latte",
      size: .large,
      temperature: .iced,
      milk: .oat,
      syrups: ["vanilla"]
    )

BWBCore/Sources/BWBCore/Voice/
├── Embeddings/
│   ├── MenuDocument.swift         # Document types for search
│   ├── TextEmbedder.swift         # NLEmbedding wrapper
│   ├── VectorIndex.swift          # In-memory vector search
│   └── MenuEmbeddingService.swift # Main search service
├── Slots/
│   ├── SlotDefinitions.swift      # Slot types and patterns
│   ├── SlotPrediction.swift       # Prediction types
│   └── SlotClassifier.swift       # Extraction logic
├── Dialogue/
│   ├── ClarificationPolicy.swift  # When to clarify
│   └── ConfirmationGenerator.swift # Generate confirmations
├── Constraints/
│   ├── ConstraintTypes.swift      # YAML data structures
│   ├── ConstraintParser.swift     # YAML loading
│   └── YAMLConstraintEngine.swift # Validation engine
├── EnhancedVoiceNLUEngine.swift   # Main NLU engine
├── VoiceNLUEngine.swift           # Legacy engine (still works)
├── VoiceDialogueManager.swift     # State machine
├── SpeechAnalyzerService.swift    # iOS 26 speech
└── VoiceTypes.swift               # Shared types

BWBCore/Resources/
└── cafe_constraints.yaml          # Menu constraints

struct NLUResult {
    let intent: VoiceIntent           // .order, .modify, .cancel, etc.
    let parsedOrders: [VoiceParsedOrder]
    let clarificationsNeeded: [ClarificationRequest]
    let overallConfidence: Double
    let rawTranscript: String
}

struct VoiceParsedOrder {
    var itemId: String?
    var itemName: String
    var size: DrinkSize?
    var temperature: DrinkTemperature?
    var milk: MilkType?
    var caffeine: CaffeineOption?
    var shots: Int?
    var syrups: [String]
    var quantity: Int
    var confidence: Double
}

struct ClarificationRequest {
    let slotName: String
    let question: String
    let options: [String]
    let importance: ClarificationImportance  // .high, .medium, .low
}

// In KioskOrderingView.swift
@StateObject private var nluEngine = EnhancedVoiceNLUEngine()

func handleTranscript(_ transcript: String) async {
    let result = await nluEngine.process(transcript: transcript)

    if !result.clarificationsNeeded.isEmpty {
        // Ask for clarification
        showClarification(result.clarificationsNeeded.first!)
    } else if let order = result.parsedOrders.first {
        // Add to cart
        addToCart(order)
    }
}

// Old way (still works)
let legacyEngine = VoiceNLUEngine.shared
let result = await legacyEngine.process(transcript: text)

// New way (recommended)
let enhancedEngine = EnhancedVoiceNLUEngine()
await enhancedEngine.initialize()
let result = await enhancedEngine.process(transcript: text)

Customer: "large oatmeal latte"
              ↓
System: "large ??? latte" (milk unknown)
              ↓
Barista corrects → "oat milk"
              ↓
FeedbackCollector logs: "oatmeal" → "oat"
              ↓
After 3 corrections → Suggestion generated
              ↓
Admin approves → PatternLearner applies
              ↓
Next time: "oatmeal" recognized as "oat"

// Record a correction
FeedbackCollector.shared.recordCorrection(
    originalTranscript: "large oatmeal latte",
    originalOrder: wrongOrder,
    correctedOrder: fixedOrder
)

// Review suggestions
let pending = FeedbackCollector.shared.getPendingSuggestions()

// Approve and apply
FeedbackCollector.shared.approveSuggestion(suggestion.id)
PatternLearner.shared.applyApprovedSuggestions()

// Export learned aliases
let yaml = PatternLearner.shared.exportAsYAML()

Documents/voice_feedback/
├── corrections.json      # All corrections
├── patterns.json         # Detected patterns
├── suggestions.json      # Pending suggestions
└── learned_aliases.json  # Applied aliases