Data Pipeline Consolidation - Python vs Rust

core/cc-ml/data_pipeline/
├── downloaders/
│   ├── youtube_downloader.py    # yt-dlp wrapper
│   └── music_list_processor.py  # YouTube search
├── processors/
│   └── audio_processor.py       # pydub conversion
├── storage/
│   └── local_music_database.py  # JSON database
└── pipeline/
    └── music_pipeline.py        # Orchestration

// Motion data processing
pub struct DataPipeline {
    // Processes sensor data from devices
    // Stores motion recordings
    // Handles rehearsal sessions
}

┌──────────────────────────────────────────────────────────────┐
│  PHASE 1: Download (Python)                                  │
├──────────────────────────────────────────────────────────────┤
│  Input:  Soundcloud likes list                               │
│  Tool:   core/cc-ml/data_pipeline/downloaders/               │
│  Output: Raw audio files (MP3/M4A)                           │
│                                                               │
│  Why Python:                                                 │
│  • yt-dlp is Python-native                                   │
│  • Already works (340 tracks ready)                          │
│  • No benefit to rewriting                                   │
└──────────────────────────────────────────────────────────────┘
                            ↓
                    Save to: /tmp/downloads/

┌──────────────────────────────────────────────────────────────┐
│  PHASE 2: Analysis (Rust)                                    │
├──────────────────────────────────────────────────────────────┤
│  Input:  Raw audio files                                     │
│  Tool:   cc-echelon/crates/media/analysis                    │
│  Output: AnalysisResult (BPM, energy, key, etc.)             │
│                                                               │
│  Why Rust:                                                   │
│  • Already implemented (60% done)                            │
│  • Fast batch processing                                     │
│  • Native audio decoding (symphonia)                         │
└──────────────────────────────────────────────────────────────┘
                            ↓
                    Store in: phrase_db (SQLite)

┌──────────────────────────────────────────────────────────────┐
│  PHASE 3: Organization (Rust)                                │
├──────────────────────────────────────────────────────────────┤
│  Input:  Analysis results from phrase_db                     │
│  Tool:   cc-echelon/crates/music-brain                       │
│  Output: Smart collections, playlists                        │
│                                                               │
│  Why Rust:                                                   │
│  • Fast lookups (BTreeMap)                                   │
│  • Recommendation engine already exists                      │
│  • Real-time playlist generation                             │
└──────────────────────────────────────────────────────────────┘

# Python downloads, writes manifest
python3 download_music_to_gcs.py --playlist "soundcloud_likes"
# Creates: /tmp/downloads/*.mp3 + manifest.json

# Rust reads manifest, analyzes, stores
./cc-echelon analyze --manifest /tmp/downloads/manifest.json
# Stores in: phrase_db.sqlite

comp-core/
│
├── core/cc-ml/data_pipeline/        # PYTHON (Download only)
│   ├── downloaders/
│   │   ├── youtube_downloader.py    # KEEP - uses yt-dlp
│   │   └── music_list_processor.py  # KEEP - YouTube search
│   └── cli.py                        # KEEP - download command
│
├── apps/desktop/cc-echelon/crates/  # RUST (Everything else)
│   ├── media/
│   │   ├── analysis.rs               # EXTEND - add key detection
│   │   └── phrase_db.rs              # USE - store analysis
│   │
│   ├── music-brain/                  # NEW CRATE
│   │   ├── camelot.rs                # Harmonic mixing
│   │   ├── playlist.rs               # Smart generation
│   │   └── dj_export.rs              # Rekordbox/Serato
│   │
│   └── music-cli/                    # NEW BINARY
│       └── main.rs                   # CLI for Rust pipeline
│
└── tools/music-pipeline/            # SCRIPTS (Orchestration)
    ├── download_music_to_gcs.py     # KEEP - Python wrapper
    └── analyze_and_organize.sh      # NEW - calls Rust after download

# Step 1: Download (Python)
cd tools/music-pipeline
python3 download_music_to_gcs.py soundcloud_likes.txt \
  --output-dir /tmp/music_download

# Step 2: Analyze (Rust)
cd apps/desktop/cc-echelon
cargo run --bin music-cli analyze \
  --input-dir /tmp/music_download \
  --phrase-db [home-path]

# Step 3: Generate Playlists (Rust)
cargo run --bin music-cli generate-playlist \
  --phrase-db [home-path] \
  --name "Dark Techno Set" \
  --duration 60 \
  --export rekordbox

   pub fn detect_key(samples: &[f32]) -> String {
       // Chroma feature extraction
       // Key profile matching
   }

   cd apps/desktop/cc-echelon
   cargo new --lib crates/music-brain

   // crates/music-brain/src/camelot.rs
   pub fn compatible_keys(key: &str) -> Vec<String>;

   # After download, write manifest
   manifest = {
       "tracks": [
           {"file": "track1.mp3", "artist": "...", "title": "..."},
           ...
       ]
   }
   with open("manifest.json", "w") as f:
       json.dump(manifest, f)

   // crates/music-cli/src/main.rs
   fn main() {
       let manifest = read_manifest("manifest.json");
       for track in manifest.tracks {
           let analysis = analyze_track(&track.file);
           phrase_db.store(track, analysis);
       }
   }