adding a bunch of untested files

clio-ai/real_time/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

clio-ai/real_time/.python-version

@@ -0,0 +1 @@
+3.13

clio-ai/real_time/collect_samples.sh

@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+
+# Training sample collection script for voice command classification
+# Creates organized recordings for wake word and category phrases
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RECORDINGS_DIR="$SCRIPT_DIR/recordings"
+SAMPLE_RATE=16000
+DURATION=4
+
+# Categories
+declare -A CATEGORIES=(
+    ["wake_word"]="ok, note this"
+    ["pressure_below_15"]="eye pressure is less than 15"
+    ["pressure_15_to_25"]="eye pressure is 15 to 25"
+    ["pressure_25_to_35"]="eye pressure is 25 to 35"
+    ["pressure_above_35"]="eye pressure is greater than 35"
+    ["drop_l"]="L drop"
+    ["drop_p"]="P drop"
+    ["drop_d"]="D drop"
+    ["both_eyes"]="both eyes"
+)
+
+# Get Corsair mic source
+get_mic_source() {
+    wpctl status | grep -i "corsair" | grep -i "mono" | grep -oE '[0-9]+\.' | head -1 | tr -d '.'
+}
+
+# Create directory structure
+setup_dirs() {
+    for category in "${!CATEGORIES[@]}"; do
+        mkdir -p "$RECORDINGS_DIR/$category"
+    done
+    echo "Created recording directories in: $RECORDINGS_DIR"
+}
+
+# Count existing samples for a category
+count_samples() {
+    local category=$1
+    find "$RECORDINGS_DIR/$category" -name "*.wav" 2>/dev/null | wc -l
+}
+
+# Record a single sample
+record_sample() {
+    local category=$1
+    local voice_name=$2
+    local source_id=$3
+
+    local count=$(count_samples "$category")
+    local filename="${voice_name}_$(printf "%03d" $((count + 1))).wav"
+    local filepath="$RECORDINGS_DIR/$category/$filename"
+
+    echo ""
+    echo "Recording: $category"
+    echo "Say: \"${CATEGORIES[$category]}\""
+    echo ""
+    echo "Press ENTER to start recording (${DURATION}s)..."
+    read
+
+    echo ">>> RECORDING NOW - SPEAK! <<<"
+    pw-record --target "$source_id" --rate "$SAMPLE_RATE" --channels 1 --format s16 "$filepath" &
+    local pid=$!
+    sleep "$DURATION"
+    kill "$pid" 2>/dev/null
+    wait "$pid" 2>/dev/null
+
+    # Check audio level
+    local max_vol=$(ffmpeg -i "$filepath" -af "volumedetect" -f null /dev/null 2>&1 | grep "max_volume" | sed 's/.*max_volume: \([-0-9.]*\).*/\1/')
+
+    echo "Saved: $filename (max volume: ${max_vol} dB)"
+
+    if [ "${max_vol%.*}" -lt -50 ] 2>/dev/null; then
+        echo "WARNING: Recording seems quiet. Keep? [Y/n]"
+        read -r keep
+        if [[ "$keep" =~ ^[Nn] ]]; then
+            rm "$filepath"
+            echo "Deleted. Try again."
+            return 1
+        fi
+    fi
+    return 0
+}
+
+# Interactive recording session
+record_session() {
+    local source_id=$(get_mic_source)
+
+    if [ -z "$source_id" ]; then
+        echo "ERROR: Corsair mic not found"
+        exit 1
+    fi
+
+    echo "=== Voice Command Sample Collection ==="
+    echo "Mic source: $source_id"
+    echo ""
+
+    echo "Enter your name/voice identifier (e.g., 'john', 'sarah'):"
+    read -r voice_name
+    voice_name=${voice_name:-default}
+
+    while true; do
+        echo ""
+        echo "=== Current sample counts ==="
+        for category in "${!CATEGORIES[@]}"; do
+            local count=$(count_samples "$category")
+            echo "  $category: $count samples"
+        done
+
+        echo ""
+        echo "Select category to record:"
+        echo "  1) wake_word           - \"ok, note this\""
+        echo "  2) pressure_below_15   - \"eye pressure is less than 15\""
+        echo "  3) pressure_15_to_25   - \"eye pressure is 15 to 25\""
+        echo "  4) pressure_25_to_35   - \"eye pressure is 25 to 35\""
+        echo "  5) pressure_above_35   - \"eye pressure is greater than 35\""
+        echo "  6) drop_l              - \"L drop\""
+        echo "  7) drop_p              - \"P drop\""
+        echo "  8) drop_d              - \"D drop\""
+        echo "  9) both_eyes           - \"both eyes\""
+        echo "  a) Record ALL categories (one each)"
+        echo "  q) Quit"
+        echo ""
+        read -r choice
+
+        case $choice in
+            1) record_sample "wake_word" "$voice_name" "$source_id" ;;
+            2) record_sample "pressure_below_15" "$voice_name" "$source_id" ;;
+            3) record_sample "pressure_15_to_25" "$voice_name" "$source_id" ;;
+            4) record_sample "pressure_25_to_35" "$voice_name" "$source_id" ;;
+            5) record_sample "pressure_above_35" "$voice_name" "$source_id" ;;
+            6) record_sample "drop_l" "$voice_name" "$source_id" ;;
+            7) record_sample "drop_p" "$voice_name" "$source_id" ;;
+            8) record_sample "drop_d" "$voice_name" "$source_id" ;;
+            9) record_sample "both_eyes" "$voice_name" "$source_id" ;;
+            a|A)
+                for category in wake_word pressure_below_15 pressure_15_to_25 pressure_25_to_35 pressure_above_35 drop_l drop_p drop_d both_eyes; do
+                    record_sample "$category" "$voice_name" "$source_id"
+                done
+                ;;
+            q|Q)
+                echo "Done. Total samples collected:"
+                for category in "${!CATEGORIES[@]}"; do
+                    echo "  $category: $(count_samples "$category")"
+                done
+                exit 0
+                ;;
+            *) echo "Invalid choice" ;;
+        esac
+    done
+}
+
+# Main
+setup_dirs
+record_session

clio-ai/real_time/flake.lock

@@ -0,0 +1,61 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1778443072,
+        "narHash": "sha256-zi7/fsqM/kFdNuED//4WOCUtezGtKKqRNORjMvfwjnA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "da5ad661ba4e5ef59ba743f0d112cbc30e474f32",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

clio-ai/real_time/flake.nix

@@ -0,0 +1,60 @@
+{
+  description = "Voice-activated eye pressure classifier";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          config.allowUnfree = true;
+          config.cudaSupport = true;
+        };
+      in
+      {
+        devShells.default = pkgs.mkShell {
+          buildInputs = with pkgs; [
+            # Python and uv
+            python313
+            uv
+
+            # Audio
+            portaudio
+            pipewire
+            alsa-lib
+            ffmpeg
+
+            # CUDA (for faster-whisper)
+            cudaPackages.cudatoolkit
+            cudaPackages.cudnn
+
+            # SSL certificates
+            cacert
+
+            # Text-to-speech
+            espeak-ng
+          ];
+
+          shellHook = ''
+            export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath [
+              pkgs.portaudio
+              pkgs.pipewire
+              pkgs.alsa-lib
+              pkgs.cudaPackages.cudatoolkit
+              pkgs.cudaPackages.cudnn
+            ]}:/run/opengl-driver/lib:$LD_LIBRARY_PATH"
+
+            export SSL_CERT_FILE="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+            export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
+
+            echo "Voice classifier dev environment"
+            echo "Run: uv run python listener.py"
+          '';
+        };
+      }
+    );
+}

clio-ai/real_time/listener.py

@@ -0,0 +1,562 @@
+#!/usr/bin/env python3
+"""
+Voice-activated eye pressure classifier.
+
+Listens for wake word "ok, note this" then classifies the following speech
+into eye pressure categories.
+
+Uses two classification methods:
+1. Transcription + regex pattern matching (fallback)
+2. Audio embeddings + trained classifier (primary, if available)
+
+Usage:
+    uv run python listener.py [--model large-v3] [--device cuda]
+"""
+
+import argparse
+import collections
+import pickle
+import queue
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import sounddevice as sd
+import torch
+import webrtcvad
+
+# Lazy imports for faster startup feedback
+whisper_model = None
+embedding_model = None
+classifier_data = None
+
+SCRIPT_DIR = Path(__file__).parent
+CLASSIFIER_PATH = SCRIPT_DIR / "classifier.pkl"
+
+# Categories with regex patterns for matching
+CATEGORIES = {
+    "pressure_below_15": [
+        r"(less\s+than|under|below)\s*15",
+        r"<\s*15",
+        r"15.*less",
+    ],
+    "pressure_15_to_25": [
+        r"15\s*(to|through|-)\s*25",
+        r"between\s*15\s*(and|to)\s*25",
+    ],
+    "pressure_25_to_35": [
+        r"25\s*(to|through|-)\s*35",
+        r"between\s*25\s*(and|to)\s*35",
+    ],
+    "pressure_above_35": [
+        r"(greater\s+than|over|above|more\s+than)\s*35",
+        r">\s*35",
+        r"35.*more",
+    ],
+    "drop_l": [
+        r"\bl\s*drop",
+        r"drop\s*l\b",
+        r"\bell\s*drop",
+    ],
+    "drop_p": [
+        r"\bp\s*drop",
+        r"drop\s*p\b",
+        r"\bpee\s*drop",
+    ],
+    "drop_d": [
+        r"\bd\s*drop",
+        r"drop\s*d\b",
+        r"\bdee\s*drop",
+    ],
+    "both_eyes": [
+        r"both\s*eyes",
+        r"both\s*i",
+    ],
+}
+
+WAKE_PHRASES = [
+    r"ok[,.]?\s*note\s+this",
+    r"okay[,.]?\s*note\s+this",
+    r"o\.?k\.?\s*note\s+this",
+    # Common mis-transcriptions
+    r"ok[,.]?\s*notice",
+    r"okay[,.]?\s*notice",
+    r"ok[,.]?\s*noted",
+    r"okay[,.]?\s*noted",
+    r"ok[,.]?\s*no\s+this",
+    r"okay[,.]?\s*no\s+this",
+    r"ok[,.]?\s*not\s+this",
+    r"okay[,.]?\s*not\s+this",
+]
+
+CONFIRM_YES = [
+    r"\byes\b", r"\byeah\b", r"\byep\b", r"\byup\b",
+    r"\bcorrect\b", r"\bright\b", r"\bconfirm\b",
+    r"\bsave\b", r"\bok\b", r"\bokay\b",
+]
+
+CONFIRM_NO = [
+    r"\bno\b", r"\bnope\b", r"\bnah\b",
+    r"\bwrong\b", r"\bcancel\b", r"\bdelete\b",
+    r"\bdiscard\b", r"\bredo\b",
+]
+
+# Spoken responses for each category
+CATEGORY_RESPONSES = {
+    "pressure_below_15": "Eye pressure below 15",
+    "pressure_15_to_25": "Eye pressure 15 to 25",
+    "pressure_25_to_35": "Eye pressure 25 to 35",
+    "pressure_above_35": "Eye pressure above 35",
+    "drop_l": "L drop",
+    "drop_p": "P drop",
+    "drop_d": "D drop",
+    "both_eyes": "Both eyes",
+}
+
+SAMPLE_RATE = 16000
+FRAME_DURATION_MS = 30  # webrtcvad frame size
+FRAME_SIZE = int(SAMPLE_RATE * FRAME_DURATION_MS / 1000)
+SPEECH_PADDING_MS = 300  # padding around speech
+NUM_PADDING_FRAMES = SPEECH_PADDING_MS // FRAME_DURATION_MS
+
+
+def speak(text: str):
+    """Speak text using available TTS."""
+    # Try espeak-ng, then espeak, then piper
+    for cmd in [
+        ["espeak-ng", "-v", "en-us", "-s", "150", text],
+        ["espeak", "-v", "en-us", "-s", "150", text],
+    ]:
+        try:
+            result = subprocess.run(cmd, capture_output=True, timeout=10)
+            if result.returncode == 0:
+                return
+        except FileNotFoundError:
+            continue
+        except Exception as e:
+            print(f"  TTS error: {e}")
+            return
+
+    # Fallback: use ffmpeg to generate speech-like notification
+    print(f"  (TTS unavailable - install espeak-ng)")
+
+
+def beep():
+    """Play a short beep tone."""
+    try:
+        # Generate a 100ms 800Hz beep
+        subprocess.run(
+            ["ffplay", "-nodisp", "-autoexit", "-f", "lavfi", "-i",
+             "sine=frequency=800:duration=0.1", "-loglevel", "quiet"],
+            capture_output=True,
+            timeout=2
+        )
+    except Exception:
+        pass  # Silent fail
+
+
+def get_corsair_device():
+    """Find Corsair microphone device index."""
+    devices = sd.query_devices()
+    for i, dev in enumerate(devices):
+        if "corsair" in dev["name"].lower() and dev["max_input_channels"] > 0:
+            return i
+    return None
+
+
+def load_whisper(model_name: str, device: str):
+    """Load faster-whisper model."""
+    global whisper_model
+    if whisper_model is None:
+        from faster_whisper import WhisperModel
+        print(f"Loading Whisper model '{model_name}' on {device}...")
+        compute_type = "float16" if device == "cuda" else "int8"
+        whisper_model = WhisperModel(model_name, device=device, compute_type=compute_type)
+        print("  Whisper loaded.")
+    return whisper_model
+
+
+def load_embedding_model(device: str):
+    """Load wav2vec2 model for embeddings."""
+    global embedding_model
+    if embedding_model is None:
+        from torchaudio.pipelines import WAV2VEC2_BASE
+        print(f"Loading wav2vec2 on {device}...")
+        bundle = WAV2VEC2_BASE
+        embedding_model = bundle.get_model().to(device)
+        embedding_model.eval()
+        print("  wav2vec2 loaded.")
+    return embedding_model
+
+
+def load_classifier():
+    """Load trained classifier if available."""
+    global classifier_data
+    if classifier_data is None and CLASSIFIER_PATH.exists():
+        print(f"Loading classifier from {CLASSIFIER_PATH}...")
+        with open(CLASSIFIER_PATH, 'rb') as f:
+            classifier_data = pickle.load(f)
+        print(f"  Classifier loaded. Categories: {list(classifier_data['label_encoder'].classes_)}")
+    return classifier_data
+
+
+def transcribe(audio: np.ndarray, model) -> str:
+    """Transcribe audio to text."""
+    segments, _ = model.transcribe(audio, language="en", beam_size=5)
+    return " ".join(seg.text for seg in segments).strip().lower()
+
+
+def extract_embedding(audio: np.ndarray, model, device: str) -> np.ndarray:
+    """Extract embedding from audio using wav2vec2."""
+    with torch.no_grad():
+        # Convert to torch tensor
+        waveform = torch.from_numpy(audio).unsqueeze(0).to(device)
+        features, _ = model.extract_features(waveform)
+        # Use the last layer's features, averaged over time
+        embedding = features[-1].mean(dim=1).cpu().numpy()
+    return embedding.flatten()
+
+
+def classify_with_embedding(audio: np.ndarray, emb_model, classifier, device: str) -> tuple[str, float]:
+    """Classify audio using embedding + trained classifier."""
+    embedding = extract_embedding(audio, emb_model, device)
+    embedding = embedding.reshape(1, -1)
+
+    probs = classifier['classifier'].predict_proba(embedding)[0]
+    pred_idx = probs.argmax()
+    confidence = probs[pred_idx]
+    category = classifier['label_encoder'].inverse_transform([pred_idx])[0]
+
+    return category, confidence
+
+
+def check_wake_word(text: str) -> bool:
+    """Check if text contains wake phrase."""
+    for pattern in WAKE_PHRASES:
+        if re.search(pattern, text, re.IGNORECASE):
+            return True
+    return False
+
+
+def check_confirmation(text: str) -> str | None:
+    """Check if text is a yes/no confirmation. Returns 'yes', 'no', or None."""
+    for pattern in CONFIRM_YES:
+        if re.search(pattern, text, re.IGNORECASE):
+            return "yes"
+    for pattern in CONFIRM_NO:
+        if re.search(pattern, text, re.IGNORECASE):
+            return "no"
+    return None
+
+
+def log_recording(log_file: str, category: str, timestamp: str):
+    """Append confirmed recording to log file."""
+    import csv
+    import os
+
+    file_exists = os.path.exists(log_file)
+    with open(log_file, 'a', newline='') as f:
+        writer = csv.writer(f)
+        if not file_exists:
+            writer.writerow(['timestamp', 'category'])
+        writer.writerow([timestamp, category])
+
+
+def classify_category(text: str) -> str | None:
+    """Classify text into a category. Returns None if no match."""
+    for category, patterns in CATEGORIES.items():
+        for pattern in patterns:
+            if re.search(pattern, text, re.IGNORECASE):
+                return category
+    return None
+
+
+def classify_text(text: str) -> tuple[str | None, str | None]:
+    """
+    Check for wake word and classify category from transcription.
+    Returns (category, text) or (None, None) if no wake word.
+    """
+    if not check_wake_word(text):
+        return None, None
+
+    category = classify_category(text)
+    return category if category else "unknown", text
+
+
+class VoiceActivityDetector:
+    """VAD-based speech segmentation."""
+
+    def __init__(self, aggressiveness: int = 2):
+        self.vad = webrtcvad.Vad(aggressiveness)
+        self.ring_buffer = collections.deque(maxlen=NUM_PADDING_FRAMES)
+        self.triggered = False
+        self.voiced_frames = []
+
+    def process_frame(self, frame: bytes) -> np.ndarray | None:
+        """
+        Process a frame of audio.
+        Returns complete utterance when speech ends, None otherwise.
+        """
+        is_speech = self.vad.is_speech(frame, SAMPLE_RATE)
+
+        if not self.triggered:
+            self.ring_buffer.append((frame, is_speech))
+            num_voiced = sum(1 for _, speech in self.ring_buffer if speech)
+
+            if num_voiced > 0.9 * self.ring_buffer.maxlen:
+                self.triggered = True
+                self.voiced_frames = [f for f, _ in self.ring_buffer]
+                self.ring_buffer.clear()
+        else:
+            self.voiced_frames.append(frame)
+            self.ring_buffer.append((frame, is_speech))
+            num_unvoiced = sum(1 for _, speech in self.ring_buffer if not speech)
+
+            if num_unvoiced > 0.9 * self.ring_buffer.maxlen:
+                self.triggered = False
+                audio_bytes = b"".join(self.voiced_frames)
+                self.voiced_frames = []
+                self.ring_buffer.clear()
+
+                # Convert to numpy array
+                audio = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+                return audio
+
+        return None
+
+
+def audio_callback(indata, frames, time_info, status, audio_queue):
+    """Callback for audio stream."""
+    if status:
+        print(f"Audio status: {status}", file=sys.stderr)
+    audio_queue.put(bytes(indata))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Voice-activated eye pressure classifier")
+    parser.add_argument("--model", default="base.en", help="Whisper model name")
+    parser.add_argument("--device", default="cuda", choices=["cuda", "cpu"], help="Compute device")
+    parser.add_argument("--list-devices", action="store_true", help="List audio devices and exit")
+    parser.add_argument("--no-classifier", action="store_true", help="Disable embedding classifier")
+    parser.add_argument("--confidence-threshold", type=float, default=0.6,
+                        help="Minimum confidence for embedding classifier (default: 0.6)")
+    parser.add_argument("--wake-timeout", type=float, default=5.0,
+                        help="Seconds to wait for command after wake word (default: 5.0)")
+    parser.add_argument("--confirm", action="store_true",
+                        help="Require yes/no confirmation before saving")
+    parser.add_argument("--log-file", type=str, default="recordings_log.csv",
+                        help="File to log confirmed recordings (default: recordings_log.csv)")
+    args = parser.parse_args()
+
+    if args.list_devices:
+        print(sd.query_devices())
+        return
+
+    # Find microphone
+    mic_device = get_corsair_device()
+    if mic_device is None:
+        print("Corsair microphone not found. Available devices:")
+        print(sd.query_devices())
+        sys.exit(1)
+
+    print(f"Using microphone: {sd.query_devices(mic_device)['name']}")
+    print()
+
+    # Load models
+    whisper = load_whisper(args.model, args.device)
+
+    # Load embedding classifier if available
+    classifier = None
+    emb_model = None
+    if not args.no_classifier:
+        classifier = load_classifier()
+        if classifier:
+            emb_model = load_embedding_model(args.device)
+        else:
+            print("  No classifier found. Run train_classifier.py after collecting samples.")
+
+    print()
+
+    # Setup VAD and audio queue
+    vad = VoiceActivityDetector(aggressiveness=2)
+    audio_queue = queue.Queue()
+
+    print("=" * 60)
+    print("Listening for: \"Ok, note this\"")
+    print("Then say your command (can pause between wake word and command)")
+    print("Commands: pressure <15 / 15-25 / 25-35 / >35, L/P/D drop, both eyes")
+    if classifier:
+        print(f"Using embedding classifier (confidence threshold: {args.confidence_threshold})")
+    else:
+        print("Using transcription-only mode")
+    if args.confirm:
+        print("Confirmation mode: say 'yes' to save, 'no' to discard")
+    print("Press Ctrl+C to stop")
+    print("=" * 60)
+    print()
+
+    # State machine: idle -> awaiting_command -> (awaiting_confirmation) -> idle
+    state = "idle"
+    state_time = 0
+    pending_category = None
+
+    def do_classification(utterance, text, use_embedding=True):
+        """Classify utterance. Returns (category, method) or (None, None)."""
+        final_category = None
+        method_used = None
+
+        # Try embedding classifier first
+        if use_embedding and classifier and emb_model:
+            start = time.time()
+            emb_category, confidence = classify_with_embedding(
+                utterance, emb_model, classifier, args.device
+            )
+            emb_time = time.time() - start
+
+            print(f"  Embedding: {emb_category} ({confidence:.1%}) [{emb_time:.2f}s]")
+
+            if confidence >= args.confidence_threshold:
+                final_category = emb_category
+                method_used = f"embedding ({confidence:.1%})"
+            else:
+                print(f"  Low confidence, trying transcription")
+
+        # Fall back to transcription
+        if final_category is None:
+            text_category = classify_category(text)
+            if text_category:
+                final_category = text_category
+                method_used = "transcription"
+
+        return final_category, method_used
+
+    def announce_classification(category, method, ask_confirm=False):
+        """Announce the classification result."""
+        response = CATEGORY_RESPONSES.get(category, category)
+        print()
+        print(f"  >>> {response} <<< (via {method})")
+        print()
+        if ask_confirm:
+            speak(f"{response}, correct?")
+        else:
+            speak(response)
+        return response
+
+    def save_recording(category):
+        """Save the confirmed recording."""
+        from datetime import datetime
+        timestamp = datetime.now().isoformat()
+        log_recording(args.log_file, category, timestamp)
+        print(f"  Saved to {args.log_file}")
+        beep()
+
+    try:
+        with sd.RawInputStream(
+            samplerate=SAMPLE_RATE,
+            blocksize=FRAME_SIZE,
+            device=mic_device,
+            dtype="int16",
+            channels=1,
+            callback=lambda *a: audio_callback(*a, audio_queue),
+        ):
+            while True:
+                frame = audio_queue.get()
+
+                # Check for timeouts
+                if state != "idle" and (time.time() - state_time) > args.wake_timeout:
+                    print("  (timeout)")
+                    print()
+                    state = "idle"
+                    pending_category = None
+
+                utterance = vad.process_frame(frame)
+
+                if utterance is not None and len(utterance) > SAMPLE_RATE * 0.5:  # At least 0.5s
+                    print("Processing speech...", end=" ", flush=True)
+
+                    # Transcribe
+                    start = time.time()
+                    text = transcribe(utterance, whisper)
+                    transcribe_time = time.time() - start
+
+                    print(f"({transcribe_time:.2f}s)")
+                    print(f"  Heard: \"{text}\"")
+
+                    # State: awaiting confirmation
+                    if state == "awaiting_confirmation":
+                        confirmation = check_confirmation(text)
+                        if confirmation == "yes":
+                            print("  Confirmed!")
+                            save_recording(pending_category)
+                        elif confirmation == "no":
+                            print("  Discarded.")
+                            beep()
+                        else:
+                            print("  Say 'yes' to save or 'no' to discard")
+                            state_time = time.time()  # Reset timeout
+                            print()
+                            continue
+                        state = "idle"
+                        pending_category = None
+                        print()
+                        continue
+
+                    # State: awaiting command
+                    if state == "awaiting_command":
+                        category, method = do_classification(utterance, text)
+                        if category:
+                            announce_classification(category, method, ask_confirm=args.confirm)
+                            if args.confirm:
+                                print("  Say 'yes' to save, 'no' to discard")
+                                state = "awaiting_confirmation"
+                                state_time = time.time()
+                                pending_category = category
+                            else:
+                                save_recording(category)
+                                state = "idle"
+                        else:
+                            print("  Could not classify. Try again.")
+                            state_time = time.time()  # Reset timeout
+                        print()
+                        continue
+
+                    # State: idle - check for wake word
+                    has_wake = check_wake_word(text)
+
+                    if not has_wake:
+                        print("  (no wake word detected)")
+                        print()
+                        continue
+
+                    print("  Wake word detected!")
+
+                    # Try to classify in same utterance
+                    category, method = do_classification(utterance, text)
+                    if category:
+                        announce_classification(category, method, ask_confirm=args.confirm)
+                        if args.confirm:
+                            print("  Say 'yes' to save, 'no' to discard")
+                            state = "awaiting_confirmation"
+                            state_time = time.time()
+                            pending_category = category
+                        else:
+                            save_recording(category)
+                        print()
+                        continue
+
+                    # No category found - wait for next utterance
+                    print("  Listening for command...")
+                    beep()
+                    state = "awaiting_command"
+                    state_time = time.time()
+                    print()
+
+    except KeyboardInterrupt:
+        print("\nStopped.")
+
+
+if __name__ == "__main__":
+    main()

clio-ai/real_time/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from real-time!")
+
+
+if __name__ == "__main__":
+    main()

clio-ai/real_time/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "real-time"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "faster-whisper>=1.2.1",
+    "numpy>=2.4.4",
+    "scikit-learn>=1.8.0",
+    "scipy>=1.17.1",
+    "sounddevice>=0.5.5",
+    "torch>=2.12.0",
+    "torchaudio>=2.11.0",
+    "webrtcvad>=2.0.10",
+]

clio-ai/real_time/run_listener.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+exec nix develop --command uv run python listener.py "$@"

clio-ai/real_time/test_corsair_mic.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Test script for Corsair VOID ELITE Wireless microphone
+# Uses PipeWire for audio capture
+
+OUTPUT_FILE="test_recording.wav"
+DURATION=5
+
+echo "=== Corsair VOID ELITE Microphone Test ==="
+echo ""
+
+# Get the Corsair source ID
+CORSAIR_SOURCE=$(wpctl status | grep -i "corsair" | grep -i "mono" | grep -oE '[0-9]+\.' | head -1 | tr -d '.')
+
+if [ -z "$CORSAIR_SOURCE" ]; then
+    echo "ERROR: Corsair microphone not found in PipeWire sources"
+    wpctl status | grep -A5 "Sources:"
+    exit 1
+fi
+
+echo "Found Corsair mic: Source ID $CORSAIR_SOURCE"
+echo "Duration: ${DURATION} seconds"
+echo ""
+echo "Speak into your Corsair headset microphone..."
+echo ""
+
+# Record using pw-record
+pw-record --target "$CORSAIR_SOURCE" --rate 48000 --channels 1 --format s16 "$OUTPUT_FILE" &
+PW_PID=$!
+
+sleep "$DURATION"
+kill "$PW_PID" 2>/dev/null
+wait "$PW_PID" 2>/dev/null
+
+if [ -f "$OUTPUT_FILE" ]; then
+    echo ""
+    echo "=== Recording complete ==="
+    echo "Saved to: $OUTPUT_FILE"
+    echo ""
+
+    # Show file info
+    echo "File info:"
+    ffprobe -hide_banner "$OUTPUT_FILE" 2>&1 | grep -E "Duration|Stream"
+
+    # Check if there's actual audio (not silence)
+    echo ""
+    echo "Audio levels:"
+    ffmpeg -i "$OUTPUT_FILE" -af "volumedetect" -f null /dev/null 2>&1 | grep -E "max_volume|mean_volume"
+
+    MAX_VOL=$(ffmpeg -i "$OUTPUT_FILE" -af "volumedetect" -f null /dev/null 2>&1 | grep "max_volume" | sed 's/.*max_volume: \([-0-9.]*\).*/\1/')
+
+    echo ""
+    # Check if max volume is greater than -50 dB (audible)
+    if [ "$(echo "$MAX_VOL" | cut -d. -f1)" -gt -50 ] 2>/dev/null; then
+        echo "SUCCESS: Audio detected! Max volume: ${MAX_VOL} dB"
+    else
+        echo "WARNING: Very quiet or silent recording (${MAX_VOL} dB)"
+        echo ""
+        echo "Possible causes:"
+        echo "  - Mic mute button on headset is ON (flip the switch!)"
+        echo "  - Headset is not being worn / mic boom is retracted"
+        echo "  - Volume too low in system settings"
+    fi
+
+    echo ""
+    echo "To play back: ffplay -nodisp -autoexit $OUTPUT_FILE"
+else
+    echo ""
+    echo "=== Recording FAILED ==="
+    exit 1
+fi

clio-ai/real_time/train_classifier.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+Train an audio embedding classifier on recorded samples.
+
+Uses wav2vec2 to extract embeddings, then trains a simple classifier.
+
+Usage:
+    uv run python train_classifier.py
+"""
+
+import pickle
+from pathlib import Path
+
+import numpy as np
+import torch
+from scipy.io import wavfile
+from torchaudio.pipelines import WAV2VEC2_BASE
+
+RECORDINGS_DIR = Path(__file__).parent / "recordings"
+MODEL_PATH = Path(__file__).parent / "classifier.pkl"
+SAMPLE_RATE = 16000
+
+# Categories (excluding wake_word which is handled separately)
+CATEGORIES = [
+    "pressure_below_15",
+    "pressure_15_to_25",
+    "pressure_25_to_35",
+    "pressure_above_35",
+    "drop_l",
+    "drop_p",
+    "drop_d",
+    "both_eyes",
+]
+
+
+def load_audio(path: Path) -> torch.Tensor:
+    """Load and preprocess audio file."""
+    sr, data = wavfile.read(path)
+
+    # Convert to float32
+    if data.dtype == np.int16:
+        data = data.astype(np.float32) / 32768.0
+    elif data.dtype == np.int32:
+        data = data.astype(np.float32) / 2147483648.0
+    elif data.dtype != np.float32:
+        data = data.astype(np.float32)
+
+    # Convert to mono if stereo
+    if len(data.shape) > 1:
+        data = data.mean(axis=1)
+
+    # Resample if needed
+    if sr != SAMPLE_RATE:
+        from scipy import signal
+        num_samples = int(len(data) * SAMPLE_RATE / sr)
+        data = signal.resample(data, num_samples)
+
+    # Convert to torch tensor with batch dimension
+    waveform = torch.from_numpy(data).unsqueeze(0)
+    return waveform
+
+
+def extract_embedding(waveform: torch.Tensor, model, device: str) -> np.ndarray:
+    """Extract embedding from audio using wav2vec2."""
+    with torch.no_grad():
+        waveform = waveform.to(device)
+        features, _ = model.extract_features(waveform)
+        # Use the last layer's features, averaged over time
+        embedding = features[-1].mean(dim=1).cpu().numpy()
+    return embedding.flatten()
+
+
+def load_samples():
+    """Load all training samples."""
+    samples = []
+    labels = []
+
+    for category in CATEGORIES:
+        category_dir = RECORDINGS_DIR / category
+        if not category_dir.exists():
+            print(f"Warning: {category_dir} does not exist")
+            continue
+
+        wav_files = list(category_dir.glob("*.wav"))
+        print(f"  {category}: {len(wav_files)} samples")
+
+        for wav_file in wav_files:
+            samples.append(wav_file)
+            labels.append(category)
+
+    return samples, labels
+
+
+def main():
+    print("=== Training Audio Classifier ===\n")
+
+    # Check for samples
+    if not RECORDINGS_DIR.exists():
+        print(f"Error: {RECORDINGS_DIR} does not exist")
+        print("Run ./collect_samples.sh first to record training samples")
+        return
+
+    print("Loading samples...")
+    sample_paths, labels = load_samples()
+
+    if len(sample_paths) == 0:
+        print("\nNo samples found. Run ./collect_samples.sh first.")
+        return
+
+    if len(set(labels)) < 2:
+        print(f"\nNeed samples from at least 2 categories. Found: {set(labels)}")
+        return
+
+    print(f"\nTotal: {len(sample_paths)} samples across {len(set(labels))} categories")
+
+    # Check minimum samples per category
+    from collections import Counter
+    counts = Counter(labels)
+    min_samples = min(counts.values())
+    if min_samples < 3:
+        print(f"\nWarning: Some categories have < 3 samples. More samples = better accuracy.")
+
+    # Load model
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"\nLoading wav2vec2 on {device}...")
+    bundle = WAV2VEC2_BASE
+    model = bundle.get_model().to(device)
+    model.eval()
+
+    # Extract embeddings
+    print("Extracting embeddings...")
+    embeddings = []
+    valid_labels = []
+
+    for i, (path, label) in enumerate(zip(sample_paths, labels)):
+        try:
+            waveform = load_audio(path)
+            embedding = extract_embedding(waveform, model, device)
+            embeddings.append(embedding)
+            valid_labels.append(label)
+            print(f"  [{i+1}/{len(sample_paths)}] {path.name}")
+        except Exception as e:
+            print(f"  [{i+1}/{len(sample_paths)}] {path.name} - ERROR: {e}")
+
+    if len(embeddings) < 2:
+        print("\nNot enough valid samples to train.")
+        return
+
+    X = np.array(embeddings)
+    y = np.array(valid_labels)
+
+    print(f"\nEmbedding shape: {X.shape}")
+
+    # Train classifier
+    print("\nTraining classifier...")
+    from sklearn.preprocessing import LabelEncoder
+    from sklearn.svm import SVC
+    from sklearn.model_selection import cross_val_score
+
+    label_encoder = LabelEncoder()
+    y_encoded = label_encoder.fit_transform(y)
+
+    # Use SVM with probability estimates
+    classifier = SVC(kernel='rbf', probability=True, C=1.0)
+
+    # Cross-validation if enough samples
+    if len(X) >= 4:
+        n_splits = min(5, len(X))
+        scores = cross_val_score(classifier, X, y_encoded, cv=n_splits)
+        print(f"Cross-validation accuracy: {scores.mean():.2%} (+/- {scores.std()*2:.2%})")
+
+    # Train on all data
+    classifier.fit(X, y_encoded)
+
+    # Save model
+    model_data = {
+        'classifier': classifier,
+        'label_encoder': label_encoder,
+        'categories': CATEGORIES,
+    }
+
+    with open(MODEL_PATH, 'wb') as f:
+        pickle.dump(model_data, f)
+
+    print(f"\nClassifier saved to: {MODEL_PATH}")
+    print(f"Categories: {list(label_encoder.classes_)}")
+    print("\nDone! The listener will now use this classifier.")
+
+
+if __name__ == "__main__":
+    main()

clio-ai/transcription/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

clio-ai/transcription/.python-version

@@ -0,0 +1 @@
+3.13

clio-ai/transcription/flake.lock

@@ -0,0 +1,61 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1777954456,
+        "narHash": "sha256-hGdgeU2Nk87RAuZyYjyDjFL6LK7dAZN5RE9+hrDTkDU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "549bd84d6279f9852cae6225e372cc67fb91a4c1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

clio-ai/transcription/flake.nix

@@ -0,0 +1,46 @@
+{
+  description = "Audio transcription with faster-whisper (local models only)";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        pythonEnv = pkgs.python313.withPackages (ps: with ps; [
+          pip
+        ]);
+      in
+      {
+        devShells.default = pkgs.mkShell {
+          buildInputs = with pkgs; [
+            pythonEnv
+            uv
+            ffmpeg
+            zlib
+            stdenv.cc.cc.lib
+          ];
+
+          shellHook = ''
+            export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath [
+              pkgs.zlib
+              pkgs.stdenv.cc.cc.lib
+              pkgs.ffmpeg
+            ]}:$LD_LIBRARY_PATH"
+
+            # Ensure venv exists
+            if [ ! -d .venv ]; then
+              uv venv
+            fi
+
+            echo "Transcription environment ready."
+            echo "Run: uv run python transcribe.py /path/to/recordings"
+          '';
+        };
+      }
+    );
+}

clio-ai/transcription/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from view-recordings!")
+
+
+if __name__ == "__main__":
+    main()

clio-ai/transcription/pyproject.toml

@@ -0,0 +1,10 @@
+[project]
+name = "view-recordings"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "faster-whisper>=1.2.1",
+    "openai-whisper>=20250625",
+]

clio-ai/transcription/transcribe.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""
+Transcribe audio recordings using Whisper models via faster-whisper.
+
+For Chromebook (CPU-only): Uses 'tiny' or 'base' models which run reasonably fast.
+For GPU (3090): Can use 'large-v3' for best quality.
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import timedelta
+from pathlib import Path
+
+
+def format_timestamp(seconds: float) -> str:
+    """Convert seconds to HH:MM:SS.mmm format."""
+    td = timedelta(seconds=seconds)
+    hours, remainder = divmod(td.seconds, 3600)
+    minutes, secs = divmod(remainder, 60)
+    return f"{hours:02d}:{minutes:02d}:{secs:02d}.{int(td.microseconds/1000):03d}"
+
+
+def transcribe_file(model, audio_path: Path, output_dir: Path, verbose: bool = False):
+    """Transcribe a single audio file and save results."""
+    print(f"\nTranscribing: {audio_path.name}")
+
+    segments, info = model.transcribe(
+        str(audio_path),
+        beam_size=5,
+        vad_filter=True,  # Filter out non-speech sections
+        vad_parameters=dict(
+            min_silence_duration_ms=500,
+            speech_pad_ms=200,
+        ),
+    )
+
+    print(f"  Detected language: {info.language} (probability: {info.language_probability:.2f})")
+    print(f"  Duration: {info.duration:.1f}s")
+
+    # Collect all segments
+    results = []
+    full_text = []
+
+    for segment in segments:
+        results.append({
+            "start": segment.start,
+            "end": segment.end,
+            "text": segment.text.strip(),
+        })
+        full_text.append(segment.text.strip())
+
+        if verbose:
+            print(f"  [{format_timestamp(segment.start)} -> {format_timestamp(segment.end)}] {segment.text.strip()}")
+
+    # Save outputs
+    stem = audio_path.stem
+
+    # Save JSON with timestamps
+    json_path = output_dir / f"{stem}.json"
+    with open(json_path, "w") as f:
+        json.dump({
+            "source_file": str(audio_path),
+            "language": info.language,
+            "language_probability": info.language_probability,
+            "duration": info.duration,
+            "segments": results,
+        }, f, indent=2)
+
+    # Save plain text
+    txt_path = output_dir / f"{stem}.txt"
+    with open(txt_path, "w") as f:
+        f.write("\n".join(full_text))
+
+    print(f"  Saved: {json_path.name}, {txt_path.name}")
+    print(f"  Found {len(results)} speech segments")
+
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Transcribe audio files using Whisper")
+    parser.add_argument(
+        "input_dir",
+        type=Path,
+        nargs="?",
+        default=None,
+        help="Directory containing audio files",
+    )
+    parser.add_argument(
+        "-o", "--output-dir",
+        type=Path,
+        default=None,
+        help="Output directory for transcriptions (default: input_dir/transcriptions)",
+    )
+    parser.add_argument(
+        "-m", "--model",
+        default=os.getenv("WHISPER_MODEL", "large-v3"),
+        choices=["tiny", "base", "small", "medium", "large-v3"],
+        help="Whisper model size (default: large-v3, or WHISPER_MODEL env var)",
+    )
+    parser.add_argument(
+        "--device",
+        default="auto",
+        choices=["auto", "cpu", "cuda"],
+        help="Device to use (auto detects GPU)",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Print each segment as it's transcribed",
+    )
+    parser.add_argument(
+        "--file",
+        type=Path,
+        default=None,
+        help="Transcribe a single file instead of directory",
+    )
+
+    args = parser.parse_args()
+
+    # Import here to show helpful error if not installed
+    try:
+        from faster_whisper import WhisperModel
+    except ImportError:
+        print("Error: faster-whisper not installed. Install with:")
+        print("  uv add faster-whisper")
+        print("  # or: pip install faster-whisper")
+        sys.exit(1)
+
+    # Determine device
+    if args.device == "auto":
+        import torch
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        compute_type = "float16" if device == "cuda" else "int8"
+    elif args.device == "cuda":
+        device = "cuda"
+        compute_type = "float16"
+    else:
+        device = "cpu"
+        compute_type = "int8"
+
+    print(f"Using device: {device}, compute type: {compute_type}")
+    print(f"Loading model: {args.model}")
+
+    model = WhisperModel(args.model, device=device, compute_type=compute_type)
+
+    # Handle single file or directory
+    if args.file:
+        audio_files = [args.file]
+        output_dir = args.output_dir or args.file.parent / "transcriptions"
+    elif args.input_dir:
+        input_dir = args.input_dir
+        if not input_dir.exists():
+            print(f"Error: Input directory does not exist: {input_dir}")
+            sys.exit(1)
+
+        audio_files = sorted(
+            p for p in input_dir.iterdir()
+            if p.suffix.lower() in {".ogg", ".mp3", ".wav", ".m4a", ".flac", ".webm"}
+        )
+        output_dir = args.output_dir or input_dir / "transcriptions"
+    else:
+        print("Error: Must specify either input_dir or --file")
+        sys.exit(1)
+
+    if not audio_files:
+        print("No audio files found")
+        sys.exit(1)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Found {len(audio_files)} audio file(s)")
+    print(f"Output directory: {output_dir}")
+
+    for audio_path in audio_files:
+        try:
+            transcribe_file(model, audio_path, output_dir, verbose=args.verbose)
+        except Exception as e:
+            print(f"  Error transcribing {audio_path.name}: {e}")
+
+    print("\nDone!")
+
+
+if __name__ == "__main__":
+    main()