Grand Diomande Research · Full HTML Reader

Audio Synthesis System

1. [Overview](#overview) 2. [Conductor Engine](#conductor-engine) 3. [Strudel Integration](#strudel-integration) 4. [Motion-to-Audio Mapping](#motion-to-audio-mapping) 5. [Platform Implementations](#platform-implementations) 6. [Pattern Library](#pattern-library) 7. [API Reference](#api-reference)

Embodied Trajectory Systems architecture technical paper candidate score 54 .md

Full Public Reader

Audio Synthesis System

Conductor + Strudel Integration Documentation

Version: 2.0.0
Last Updated: December 26, 2024

---

Table of Contents

1. [Overview](#overview)
2. [Conductor Engine](#conductor-engine)
3. [Strudel Integration](#strudel-integration)
4. [Motion-to-Audio Mapping](#motion-to-audio-mapping)
5. [Platform Implementations](#platform-implementations)
6. [Pattern Library](#pattern-library)
7. [API Reference](#api-reference)

---

1. Overview

The Audio Synthesis System transforms real-time motion data into generative music through a multi-layer architecture:

┌─────────────────────────────────────────────────────────────────────────────┐
│                     AUDIO SYNTHESIS ARCHITECTURE                             │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                              │
│  ┌───────────────────────────────────────────────────────────────────────┐ │
│  │                         MOTION INPUT                                   │ │
│  │                                                                        │ │
│  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────────────────────┐   │ │
│  │  │  MediaPipe  │  │   Mocopi    │  │      Fused State            │   │ │
│  │  │   Pose      │  │  Skeleton   │  │                             │   │ │
│  │  └──────┬──────┘  └──────┬──────┘  └─────────────┬───────────────┘   │ │
│  │         └────────────────┴───────────────────────┘                    │ │
│  └───────────────────────────────────────┬───────────────────────────────┘ │
│                                          │                                  │
│                                          ▼                                  │
│  ┌───────────────────────────────────────────────────────────────────────┐ │
│  │                     FEATURE EXTRACTION                                 │ │
│  │                                                                        │ │
│  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  │ │
│  │  │   Energy    │  │   Gesture   │  │    Limb     │  │   Tempo     │  │ │
│  │  │   Level     │  │  Detection  │  │  Positions  │  │  Detector   │  │ │
│  │  │  (0-1)      │  │             │  │             │  │  (BPM)      │  │ │
│  │  └──────┬──────┘  └──────┬──────┘  └──────┬──────┘  └──────┬──────┘  │ │
│  └─────────┼────────────────┼────────────────┼────────────────┼─────────┘ │
│            │                │                │                │            │
│            └────────────────┴────────────────┴────────────────┘            │
│                                    │                                        │
│                                    ▼                                        │
│  ┌───────────────────────────────────────────────────────────────────────┐ │
│  │                     CONDUCTOR ENGINE                                   │ │
│  │                                                                        │ │
│  │  ┌─────────────────────────────────────────────────────────────────┐ │ │
│  │  │                    Section State Machine                         │ │ │
│  │  │                                                                  │ │ │
│  │  │   IDLE ──▶ BUILDUP ──▶ DROP ──▶ BREAKDOWN ──▶ OUTRO ──▶ IDLE   │ │ │
│  │  │                                                                  │ │ │
│  │  └──────────────────────────────┬──────────────────────────────────┘ │ │
│  │                                 │                                     │ │
│  │  ┌─────────────┐  ┌─────────────┴─────────────┐  ┌─────────────────┐ │ │
│  │  │  Trajectory │  │      Audio Parameter      │  │    Pattern      │ │ │
│  │  │  Predictor  │  │        Mapping            │  │    Selector     │ │ │
│  │  └──────┬──────┘  └──────────────┬────────────┘  └────────┬────────┘ │ │
│  └─────────┼────────────────────────┼────────────────────────┼──────────┘ │
│            │                        │                        │             │
│            └────────────────────────┴────────────────────────┘             │
│                                    │                                        │
│                                    ▼                                        │
│  ┌───────────────────────────────────────────────────────────────────────┐ │
│  │                     STRUDEL ENGINE                                     │ │
│  │                                                                        │ │
│  │  ┌─────────────────────────────────────────────────────────────────┐ │ │
│  │  │                    Pattern Evaluation                            │ │ │
│  │  │                                                                  │ │ │
│  │  │   s("bd sd bd sd").lpf(cutoff).gain(volume).pan(position)       │ │ │
│  │  │                                                                  │ │ │
│  │  └──────────────────────────────┬──────────────────────────────────┘ │ │
│  │                                 │                                     │ │
│  │  ┌─────────────┐  ┌─────────────┴─────────────┐  ┌─────────────────┐ │ │
│  │  │   Sample    │  │      Web Audio API        │  │    Effects      │ │ │
│  │  │   Library   │  │     (AudioContext)        │  │    Chain        │ │ │
│  │  └─────────────┘  └───────────────────────────┘  └─────────────────┘ │ │
│  └───────────────────────────────────────────────────────────────────────┘ │
│                                    │                                        │
│                                    ▼                                        │
│                              🔊 Audio Output                                │
│                                                                              │
└─────────────────────────────────────────────────────────────────────────────┘

---

2. Conductor Engine

2.1 Overview

The Conductor Engine orchestrates musical sections based on motion energy trajectories.

Location: `apps/web/cc-dashboard/src/lib/conductor/`

2.2 Section State Machine

typescript
// Location: apps/web/cc-dashboard/src/lib/conductor/SectionState.ts

export type SectionType =
  | 'idle'      // No motion, ambient/silence
  | 'buildup'   // Energy increasing, tension building
  | 'drop'      // Peak energy, full arrangement
  | 'breakdown' // Energy decreasing, elements stripped
  | 'outro'     // Motion stopping, fade out

export interface SectionConfig {
  type: SectionType
  minDuration: number    // Minimum time in section (ms)
  maxDuration: number    // Maximum before auto-transition
  energyThreshold: {
    enter: number        // Energy to enter this section
    exit: number         // Energy to exit this section
  }
  patterns: string[]     // Available patterns for section
  effects: EffectConfig[]
}

export const DEFAULT_SECTIONS: Record<SectionType, SectionConfig> = {
  idle: {
    type: 'idle',
    minDuration: 2000,
    maxDuration: Infinity,
    energyThreshold: { enter: 0, exit: 0.2 },
    patterns: ['ambient_pad', 'silence'],
    effects: [
      { type: 'reverb', wet: 0.8 },
      { type: 'lpf', cutoff: 400 }
    ]
  },
  buildup: {
    type: 'buildup',
    minDuration: 4000,
    maxDuration: 16000,
    energyThreshold: { enter: 0.2, exit: 0.7 },
    patterns: ['buildup_drums', 'rising_synth'],
    effects: [
      { type: 'hpf', cutoff: { from: 100, to: 2000 } },
      { type: 'gain', value: { from: 0.3, to: 0.8 } }
    ]
  },
  drop: {
    type: 'drop',
    minDuration: 8000,
    maxDuration: 32000,
    energyThreshold: { enter: 0.7, exit: 0.4 },
    patterns: ['full_beat', 'bass_heavy', 'lead_synth'],
    effects: [
      { type: 'distortion', amount: 0.2 },
      { type: 'gain', value: 1.0 }
    ]
  },
  breakdown: {
    type: 'breakdown',
    minDuration: 4000,
    maxDuration: 16000,
    energyThreshold: { enter: 0.4, exit: 0.2 },
    patterns: ['minimal_drums', 'atmospheric'],
    effects: [
      { type: 'lpf', cutoff: { from: 8000, to: 800 } },
      { type: 'reverb', wet: { from: 0.2, to: 0.6 } }
    ]
  },
  outro: {
    type: 'outro',
    minDuration: 4000,
    maxDuration: 8000,
    energyThreshold: { enter: 0.2, exit: 0 },
    patterns: ['fade_drums', 'ambient_outro'],
    effects: [
      { type: 'lpf', cutoff: { from: 800, to: 200 } },
      { type: 'gain', value: { from: 0.5, to: 0 } }
    ]
  }
}

2.3 State Transitions

                         ┌──────────────────────────────────────┐
                         │         STATE TRANSITIONS            │
                         └──────────────────────────────────────┘

     motion detected              energy > 0.7               sustain < 8s
          (E > 0.2)                                              or
             │                        │                      energy < 0.4
             ▼                        ▼                          │
  ┌──────────────────┐     ┌──────────────────┐     ┌──────────────────┐
  │                  │     │                  │     │                  │
  │      IDLE        │────▶│     BUILDUP      │────▶│       DROP       │
  │                  │     │                  │     │                  │
  └──────────────────┘     └──────────────────┘     └──────────────────┘
           ▲                        │                        │
           │                        │ energy < 0.3           │
           │                        ▼                        ▼
           │               ┌──────────────────┐     ┌──────────────────┐
           │               │                  │     │                  │
           │               │      OUTRO       │◀────│    BREAKDOWN     │
           │               │                  │     │                  │
           │               └────────┬─────────┘     └──────────────────┘
           │                        │
           │    motion stops        │
           │      (E < 0.1)         │
           └────────────────────────┘


TRANSITION CONDITIONS:

  IDLE → BUILDUP:
    - Energy rises above 0.2
    - Movement detected for > 500ms

  BUILDUP → DROP:
    - Energy rises above 0.7
    - OR buildup duration > 16s

  DROP → BREAKDOWN:
    - Energy falls below 0.4
    - OR drop duration > 32s

  BREAKDOWN → OUTRO:
    - Energy falls below 0.2
    - OR breakdown duration > 16s

  OUTRO → IDLE:
    - Energy falls below 0.1
    - OR outro duration > 8s

  ANY → IDLE (emergency):
    - No motion for > 5s
    - Manual stop command

2.4 Trajectory Predictor

typescript
// Location: apps/web/cc-dashboard/src/lib/conductor/TrajectoryPredictor.ts

export class TrajectoryPredictor {
  private history: EnergyPoint[]
  private windowSize: number = 60  // 2 seconds at 30fps

  /**
   * Predict future energy trajectory using linear regression
   */
  predict(lookahead: number = 30): Prediction {
    if (this.history.length < 10) {
      return { trend: 'stable', confidence: 0, predictedEnergy: 0.5 }
    }

    // Linear regression on recent history
    const recent = this.history.slice(-this.windowSize)
    const { slope, intercept, r2 } = this.linearRegression(recent)

    // Predict future energy
    const futureTime = recent.length + lookahead
    const predictedEnergy = Math.max(0, Math.min(1, slope * futureTime + intercept))

    // Determine trend
    let trend: 'rising' | 'falling' | 'stable'
    if (slope > 0.01) trend = 'rising'
    else if (slope < -0.01) trend = 'falling'
    else trend = 'stable'

    return {
      trend,
      confidence: r2,
      predictedEnergy,
      slope,
      timeToThreshold: this.estimateTimeToThreshold(slope, intercept)
    }
  }

  /**
   * Estimate time until energy crosses key thresholds
   */
  private estimateTimeToThreshold(
    slope: number,
    intercept: number
  ): Record<string, number | null> {
    const currentEnergy = this.history[this.history.length - 1]?.energy ?? 0.5
    const thresholds = [0.2, 0.4, 0.7, 0.9]

    return Object.fromEntries(
      thresholds.map(threshold => {
        if (slope === 0) return [threshold, null]

        const framesUntil = (threshold - currentEnergy) / slope
        if (framesUntil < 0) return [threshold, null]

        return [threshold, framesUntil / 30]  // Convert to seconds
      })
    )
  }
}

2.5 Conductor Engine Implementation

typescript
// Location: apps/web/cc-dashboard/src/lib/conductor/ConductorEngine.ts

export class ConductorEngine extends EventEmitter<ConductorEvents> {
  private state: ConductorState
  private sectionManager: SectionManager
  private trajectoryPredictor: TrajectoryPredictor
  private strudelBridge: StrudelBridge

  private animationFrame: number | null = null

  constructor(config: ConductorConfig) {
    super()

    this.state = {
      currentSection: 'idle',
      energy: 0,
      tempo: 120,
      activePatterns: [],
      parameters: new Map()
    }

    this.sectionManager = new SectionManager(config.sections)
    this.trajectoryPredictor = new TrajectoryPredictor()
    this.strudelBridge = new StrudelBridge(config.strudel)
  }

  /**
   * Update conductor with new motion state
   */
  update(motionState: MotionState): void {
    // Extract features
    const energy = this.extractEnergy(motionState)
    const gestures = this.extractGestures(motionState)
    const limbPositions = this.extractLimbPositions(motionState)

    // Update state
    this.state.energy = energy

    // Add to trajectory predictor
    this.trajectoryPredictor.addPoint({
      time: Date.now(),
      energy
    })

    // Check for section transitions
    const prediction = this.trajectoryPredictor.predict()
    const newSection = this.sectionManager.evaluate(
      this.state.currentSection,
      energy,
      prediction
    )

    if (newSection !== this.state.currentSection) {
      this.transitionToSection(newSection)
    }

    // Map motion to audio parameters
    const params = this.mapMotionToAudio(motionState, limbPositions, gestures)

    // Update Strudel
    this.strudelBridge.setParameters(params)
    this.strudelBridge.setPatterns(this.state.activePatterns)

    // Emit update event
    this.emit('update', {
      section: this.state.currentSection,
      energy,
      prediction,
      parameters: params
    })
  }

  /**
   * Transition to a new section
   */
  private transitionToSection(section: SectionType): void {
    const oldSection = this.state.currentSection
    this.state.currentSection = section

    // Get section config
    const config = this.sectionManager.getConfig(section)

    // Update patterns
    this.state.activePatterns = this.selectPatterns(config.patterns)

    // Apply effects
    this.strudelBridge.applyEffects(config.effects)

    // Emit transition event
    this.emit('transition', {
      from: oldSection,
      to: section,
      patterns: this.state.activePatterns
    })
  }

  /**
   * Map motion features to audio parameters
   */
  private mapMotionToAudio(
    motion: MotionState,
    limbs: LimbPositions,
    gestures: DetectedGestures
  ): AudioParameters {
    return {
      // Filter controlled by arm height
      filterCutoff: this.mapRange(
        (limbs.leftArm.height + limbs.rightArm.height) / 2,
        0, 1,
        200, 8000
      ),

      // Volume controlled by body energy
      masterGain: this.mapRange(
        this.state.energy,
        0, 1,
        0.3, 1.0
      ),

      // Pan controlled by body lean
      pan: this.mapRange(
        motion.bodyLean,
        -1, 1,
        -0.8, 0.8
      ),

      // Reverb controlled by arm spread
      reverbWet: this.mapRange(
        limbs.armSpread,
        0, 1,
        0.1, 0.6
      ),

      // Distortion controlled by crouch
      distortion: this.mapRange(
        motion.crouchLevel,
        0, 1,
        0, 0.4
      ),

      // Tempo variation controlled by leg energy
      tempoMultiplier: this.mapRange(
        motion.lowerBodyEnergy,
        0, 1,
        0.9, 1.1
      ),

      // Pattern variations from gestures
      patternVariation: this.mapGesturesToVariation(gestures)
    }
  }

  /**
   * Start the conductor loop
   */
  start(): void {
    if (this.animationFrame) return

    const loop = () => {
      // Loop runs even without motion input
      // (for ambient/decay handling)
      this.animationFrame = requestAnimationFrame(loop)
    }

    this.strudelBridge.start()
    loop()

    this.emit('start')
  }

  /**
   * Stop the conductor
   */
  stop(): void {
    if (this.animationFrame) {
      cancelAnimationFrame(this.animationFrame)
      this.animationFrame = null
    }

    this.strudelBridge.stop()
    this.transitionToSection('idle')

    this.emit('stop')
  }
}

---

3. Strudel Integration

3.1 Overview

Strudel is a JavaScript port of TidalCycles, providing live-coding pattern-based music synthesis.

3.2 Pattern Language

javascript
// Basic patterns
s("bd sd bd sd")           // Four-on-floor kick-snare
s("bd hh sd hh")           // Standard rock beat
s("bd*4")                  // Kick on every beat
s("bd [hh hh] sd [hh oh]") // With subdivisions

// Effects
.lpf(2000)                 // Low-pass filter at 2000Hz
.hpf(100)                  // High-pass filter
.gain(0.8)                 // Volume
.pan(0.3)                  // Stereo position
.delay(0.25)               // Delay effect
.reverb(0.4)               // Reverb amount
.distort(0.2)              // Distortion

// Modulation
.lpf(sine.range(200, 4000))  // LFO on filter
.gain(perlin.range(0.5, 1))  // Perlin noise on volume

// Combining patterns
stack(
  s("bd sd bd sd"),
  s("hh*8"),
  note("c3 eb3 g3 bb3").sound("sawtooth")
)

3.3 Motion-Controlled Patterns

typescript
// Location: apps/web/cc-dashboard/src/lib/strudel/MotionStrudel.ts

export class MotionStrudel {
  private repl: Repl
  private currentParams: AudioParameters

  constructor() {
    this.repl = new Repl()
    this.currentParams = DEFAULT_PARAMS
  }

  /**
   * Generate pattern based on section and parameters
   */
  generatePattern(section: SectionType, params: AudioParameters): string {
    const template = SECTION_TEMPLATES[section]

    // Build pattern with motion-controlled values
    return template
      .replace('$CUTOFF', params.filterCutoff.toFixed(0))
      .replace('$GAIN', params.masterGain.toFixed(2))
      .replace('$PAN', params.pan.toFixed(2))
      .replace('$REVERB', params.reverbWet.toFixed(2))
      .replace('$DISTORT', params.distortion.toFixed(2))
      .replace('$TEMPO', (120 * params.tempoMultiplier).toFixed(0))
  }

  /**
   * Update pattern in real-time
   */
  async update(pattern: string): Promise<void> {
    try {
      await this.repl.evaluate(pattern)
    } catch (error) {
      console.error('Pattern evaluation error:', error)
    }
  }

  /**
   * Start playback
   */
  start(): void {
    this.repl.start()
  }

  /**
   * Stop playback
   */
  stop(): void {
    this.repl.stop()
  }
}

// Section pattern templates
const SECTION_TEMPLATES: Record<SectionType, string> = {
  idle: `
    stack(
      note("c2").sound("pad").lpf($CUTOFF).reverb(0.8).gain(0.2)
    ).slow(4)
  `,

  buildup: `
    stack(
      s("bd*4").lpf($CUTOFF).gain($GAIN),
      s("~ hh*2 ~ hh*4").gain(0.4),
      note("c3 eb3 g3 bb3").sound("sawtooth")
        .lpf(sine.range(400, $CUTOFF))
        .gain($GAIN * 0.5)
    ).cpm($TEMPO / 2)
  `,

  drop: `
    stack(
      s("bd sd:2 bd sd:2").distort($DISTORT).gain($GAIN),
      s("hh*8").gain(0.6).pan(sine.range(-0.3, 0.3)),
      note("c2 c2 eb2 g2").sound("bass")
        .lpf($CUTOFF)
        .distort($DISTORT * 0.5)
        .gain($GAIN * 0.8),
      note("c4 eb4 g4 bb4").sound("supersaw")
        .lpf($CUTOFF)
        .reverb($REVERB)
        .gain($GAIN * 0.6)
    ).cpm($TEMPO / 2)
  `,

  breakdown: `
    stack(
      s("bd ~ ~ ~").gain($GAIN * 0.6),
      s("~ hh ~ hh").gain(0.3).reverb(0.5),
      note("c3 ~ eb3 ~").sound("pad")
        .lpf($CUTOFF)
        .reverb($REVERB)
        .gain($GAIN * 0.4)
    ).slow(2).cpm($TEMPO / 2)
  `,

  outro: `
    stack(
      note("c2").sound("pad")
        .lpf($CUTOFF)
        .reverb(0.8)
        .gain($GAIN * 0.3)
    ).slow(8)
  `
}

3.4 Real-Time Parameter Control

typescript
// Continuous parameter updates without pattern recompilation

export class StrudelBridge {
  private repl: Repl
  private parameterNodes: Map<string, AudioParam>

  /**
   * Set real-time parameter (doesn't recompile pattern)
   */
  setParameter(name: string, value: number): void {
    const node = this.parameterNodes.get(name)
    if (node) {
      // Smooth transition to avoid clicks
      node.linearRampToValueAtTime(
        value,
        this.repl.audioContext.currentTime + 0.05
      )
    }
  }

  /**
   * Batch set multiple parameters
   */
  setParameters(params: AudioParameters): void {
    this.setParameter('cutoff', params.filterCutoff)
    this.setParameter('gain', params.masterGain)
    this.setParameter('pan', params.pan)
    this.setParameter('reverb', params.reverbWet)
    this.setParameter('distort', params.distortion)
  }
}

---

4. Motion-to-Audio Mapping

4.1 Mapping Table

Motion FeatureAudio ParameterRangeBehavior
`armHeight` (avg)`filterCutoff`200-8000 HzHigher arms = brighter sound
`bodyEnergy``masterGain`0.3-1.0More energy = louder
`bodyLean``pan`-0.8 to 0.8Lean left/right = pan
`armSpread``reverbWet`0.1-0.6Wide arms = more reverb
`crouchLevel``distortion`0-0.4Crouch = distortion
`lowerBodyEnergy``tempoMultiplier`0.9-1.1Leg movement = tempo
`handGesture``patternVariation`(see below)Gesture triggers

4.2 Gesture-to-Pattern Mapping

typescript
const GESTURE_MAPPINGS: Record<HandGesture, PatternModification> = {
  'open': {
    description: 'Open hands - full arrangement',
    patterns: ['full_beat', 'lead_synth'],
    parameters: { gain: 1.0 }
  },

  'fist': {
    description: 'Fists - aggressive/distorted',
    patterns: ['heavy_beat'],
    parameters: { distortion: 0.4, gain: 0.9 }
  },

  'point': {
    description: 'Pointing - accent/stab',
    patterns: ['stab_synth'],
    parameters: { attack: 0.01 }
  },

  'peace': {
    description: 'Peace sign - melodic elements',
    patterns: ['melody_arp'],
    parameters: { reverb: 0.4 }
  },

  'thumbsUp': {
    description: 'Thumbs up - filter sweep up',
    patterns: ['riser'],
    parameters: { filterDirection: 'up' }
  },

  'pinch': {
    description: 'Pinch - fine control mode',
    patterns: ['minimal'],
    parameters: { controlMode: 'fine' }
  }
}

4.3 Mapping Functions

typescript
// Location: apps/web/cc-dashboard/src/lib/conductor/ConductorEngine.ts

/**
 * Map motion value to audio range with optional curve
 */
function mapRange(
  value: number,
  inMin: number,
  inMax: number,
  outMin: number,
  outMax: number,
  curve: 'linear' | 'exponential' | 'logarithmic' = 'linear'
): number {
  // Clamp input
  const clamped = Math.max(inMin, Math.min(inMax, value))

  // Normalize to 0-1
  const normalized = (clamped - inMin) / (inMax - inMin)

  // Apply curve
  let curved: number
  switch (curve) {
    case 'exponential':
      curved = normalized * normalized
      break
    case 'logarithmic':
      curved = Math.sqrt(normalized)
      break
    default:
      curved = normalized
  }

  // Map to output range
  return outMin + curved * (outMax - outMin)
}

/**
 * Extract energy from motion state
 */
function extractEnergy(motion: MotionState): number {
  const weights = {
    upperBody: 0.4,
    lowerBody: 0.3,
    arms: 0.2,
    head: 0.1
  }

  return (
    motion.upperBodyEnergy * weights.upperBody +
    motion.lowerBodyEnergy * weights.lowerBody +
    ((motion.leftArmEnergy + motion.rightArmEnergy) / 2) * weights.arms +
    motion.headEnergy * weights.head
  )
}

---

5. Platform Implementations

5.1 Web (Dashboard)

typescript
// Location: apps/web/cc-dashboard/src/lib/strudel/

// Main Strudel integration for web
export { MotionStrudel } from './MotionStrudel'
export { PatternTemplates, SECTION_TEMPLATES } from './PatternTemplates'

5.2 iOS (EchelonCapture)

swift
// Location: apps/ios/EchelonCapture/EchelonCapture/Services/StrudelEngine.swift

import WebKit

class StrudelEngine: NSObject, ObservableObject {
    @Published var isPlaying = false
    @Published var currentPattern = ""

    private var webView: WKWebView?
    private let strudelHTML: String

    init() {
        // Load embedded Strudel HTML
        if let path = Bundle.main.path(forResource: "strudel-engine", ofType: "html"),
           let html = try? String(contentsOfFile: path) {
            strudelHTML = html
        } else {
            strudelHTML = ""
        }
        super.init()
        setupWebView()
    }

    private func setupWebView() {
        let config = WKWebViewConfiguration()
        config.allowsInlineMediaPlayback = true

        webView = WKWebView(frame: .zero, configuration: config)
        webView?.loadHTMLString(strudelHTML, baseURL: nil)
    }

    func setParameters(_ params: [String: Double]) {
        let json = try? JSONSerialization.data(withJSONObject: params)
        if let jsonString = String(data: json!, encoding: .utf8) {
            let js = "window.setParameters(\(jsonString))"
            webView?.evaluateJavaScript(js, completionHandler: nil)
        }
    }

    func play() {
        webView?.evaluateJavaScript("window.startAudio()", completionHandler: nil)
        isPlaying = true
    }

    func stop() {
        webView?.evaluateJavaScript("window.stopAudio()", completionHandler: nil)
        isPlaying = false
    }
}

5.3 Desktop (Tauri/Echelon)

javascript
// Location: apps/desktop/cc-echelon/apps/echelon-tauri/src/audio/StrudelEngine.js

import { repl, controls } from '@strudel/repl'

class StrudelEngine {
  constructor() {
    this.repl = null
    this.isPlaying = false
    this.parameters = {}
  }

  async initialize() {
    this.repl = repl({
      defaultOutput: controls({ gain: 0.5 }),
      getTime: () => performance.now() / 1000,
      transpiler: 'js'
    })
  }

  setPattern(pattern) {
    if (this.repl) {
      this.repl.evaluate(pattern)
    }
  }

  setParameter(name, value) {
    this.parameters[name] = value
    // Update pattern with new parameters
    this.rebuildPattern()
  }

  start() {
    if (this.repl) {
      this.repl.start()
      this.isPlaying = true
    }
  }

  stop() {
    if (this.repl) {
      this.repl.stop()
      this.isPlaying = false
    }
  }
}

export default new StrudelEngine()

---

6. Pattern Library

6.1 Drum Patterns

javascript
// Basic patterns
const DRUM_PATTERNS = {
  // Four-on-floor
  fourOnFloor: `s("bd*4")`,

  // Standard rock
  rockBeat: `s("bd sd bd sd")`,

  // House
  house: `stack(
    s("bd*4"),
    s("~ hh*2 ~ hh*2"),
    s("~ ~ sd ~")
  )`,

  // Breakbeat
  breakbeat: `s("bd ~ [~ bd] ~ sd ~ [bd bd] ~")`,

  // DnB
  dnb: `stack(
    s("bd ~ ~ ~ bd ~ ~ ~"),
    s("~ ~ sd ~ ~ ~ sd ~"),
    s("hh*16")
  ).fast(2)`,

  // Trap
  trap: `stack(
    s("bd ~ ~ bd ~ ~ bd ~"),
    s("~ ~ ~ ~ sd ~ ~ ~"),
    s("hh*8").gain(0.4)
  )`,

  // Minimal techno
  minimalTechno: `stack(
    s("bd*4"),
    s("[~ hh]*4").gain(0.3),
    s("~ sd:3 ~ ~")
  )`
}

6.2 Synth Patterns

javascript
const SYNTH_PATTERNS = {
  // Bass
  acidBass: `note("c2 c2 eb2 g2")
    .sound("sawtooth")
    .lpf(sine.range(200, 2000))
    .resonance(0.8)`,

  subBass: `note("c1")
    .sound("sine")
    .lpf(100)
    .gain(0.8)`,

  // Leads
  supersaw: `note("c4 eb4 g4 bb4")
    .sound("supersaw")
    .detune(0.2)
    .lpf(4000)`,

  pluck: `note("c5 g4 eb5 bb4")
    .sound("pluck")
    .decay(0.1)`,

  // Pads
  warmPad: `note("c3 eb3 g3")
    .sound("pad")
    .attack(0.5)
    .release(2)
    .lpf(2000)
    .reverb(0.6)`,

  // Arps
  arp: `note("c4 eb4 g4 bb4 c5 bb4 g4 eb4")
    .sound("triangle")
    .lpf(2000)
    .delay(0.25)`
}

6.3 Effect Chains

javascript
const EFFECT_CHAINS = {
  // Clean
  clean: `.gain(0.8)`,

  // Warm
  warm: `.lpf(4000).distort(0.1).reverb(0.2)`,

  // Spacey
  spacey: `.reverb(0.6).delay(0.33).lpf(6000)`,

  // Aggressive
  aggressive: `.distort(0.4).hpf(200).gain(0.9)`,

  // Lo-fi
  lofi: `.lpf(2000).distort(0.1).crush(8)`,

  // Underwater
  underwater: `.lpf(sine.range(200, 800)).reverb(0.8).gain(0.5)`
}

---

7. API Reference

7.1 ConductorEngine API

typescript
// Constructor
new ConductorEngine(config: ConductorConfig)

// Methods
update(motionState: MotionState): void
start(): void
stop(): void
setSection(section: SectionType): void
setTempo(bpm: number): void

// Events
on('update', (state: ConductorUpdate) => void): void
on('transition', (transition: SectionTransition) => void): void
on('start', () => void): void
on('stop', () => void): void

// Properties (read-only)
readonly currentSection: SectionType
readonly energy: number
readonly tempo: number
readonly activePatterns: string[]

7.2 StrudelBridge API

typescript
// Constructor
new StrudelBridge(config: StrudelConfig)

// Methods
setPattern(pattern: string): Promise<void>
setParameters(params: AudioParameters): void
setParameter(name: string, value: number): void
applyEffects(effects: EffectConfig[]): void
start(): void
stop(): void

// Properties
readonly isPlaying: boolean
readonly audioContext: AudioContext

7.3 Zustand Store

typescript
// Location: apps/web/cc-dashboard/src/store/conductorStore.ts

interface ConductorStore {
  // State
  isActive: boolean
  currentSection: SectionType
  energy: number
  tempo: number
  parameters: AudioParameters
  prediction: Prediction | null

  // Actions
  setActive: (active: boolean) => void
  updateFromMotion: (motion: MotionState) => void
  setSection: (section: SectionType) => void
  setTempo: (bpm: number) => void
}

export const useConductorStore = create<ConductorStore>((set, get) => ({
  isActive: false,
  currentSection: 'idle',
  energy: 0,
  tempo: 120,
  parameters: DEFAULT_PARAMS,
  prediction: null,

  setActive: (active) => set({ isActive: active }),

  updateFromMotion: (motion) => {
    const conductor = getConductorEngine()
    conductor.update(motion)
  },

  setSection: (section) => set({ currentSection: section }),
  setTempo: (tempo) => set({ tempo })
}))

---

Document Version: 2.0.0
Generated: December 26, 2024

Promotion Decision

Promote into a technical note or architecture paper with implementation anchors.

Source Anchor

projects/Documentation/01-architecture/systems/AUDIO_SYNTHESIS_SYSTEM.md

Detected Structure

Method · Evaluation · Code Anchors · Architecture