Add Audio Command Controller Streamlit application

AudioCommandController/.env.example

@@ -0,0 +1,21 @@
+# OpenAI (optional; used for smarter chat/intent extraction if set)
+OPENAI_API_KEY=sk-...
+
+# Spotify OAuth
+SPOTIFY_CLIENT_ID=your_spotify_client_id
+SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+# Add both http://localhost:8501 and your HTTPS tunnel URL in Spotify dashboard
+SPOTIFY_REDIRECT_URI=http://localhost:8501
+SPOTIFY_SCOPES=user-read-playback-state user-read-currently-playing user-modify-playback-state playlist-modify-public playlist-modify-private
+
+# MongoDB (optional persistence for Mode A chat)
+MONGO_URI=mongodb://localhost:27017
+MONGO_DB=acc
+MONGO_COLLECTION=mode_a_history
+
+# OSC target (Mode A chords/tempo; Mode B mirror)
+OSC_HOST=127.0.0.1
+OSC_PORT=7000
+
+# MIDI out port name (Windows: create LoopMIDI port; macOS: IAC)
+MIDI_PORT=ACC Port

AudioCommandController/.streamlit/config.toml

@@ -0,0 +1,12 @@
+[server]
+port = 8501
+address = "0.0.0.0"
+enableCORS = true
+headless = true
+
+[theme]
+primaryColor = "#7C4DFF"
+backgroundColor = "#0E0F12"
+secondaryBackgroundColor = "#16181D"
+textColor = "#EDEDED"
+font = "sans serif"

AudioCommandController/README.md

@@ -0,0 +1,37 @@
+# Audio Command Controller (Streamlit)
+
+Two-mode web app for iPad/Chrome:
+- **Mode A: MusicBuddy** (3 buddies; one **Spotify Buddy**). Chat + docs, Suno stub, chord/tempo to OSC, optional MongoDB.
+- **Mode B: Controller**. Conversational, 16 channels, transport, plugin stubs per channel, OSC mirror, MIDI out, clarifying questions.
+
+## Windows Quickstart
+1. Install Python 3.11+  
+2. Create a LoopMIDI virtual port named `ACC Port`.
+3. `python -m venv .venv && \.venv\\Scripts\\activate`
+4. `pip install --upgrade pip && pip install -r requirements.txt`
+5. Copy `.env.example` to `.env` and fill keys.
+6. `streamlit run app.py --server.port 8501 --server.address 0.0.0.0`
+
+### iPad mic (HTTPS)
+- `cloudflared tunnel --url http://localhost:8501` (or `ngrok http 8501`)
+- Open tunnel URL on iPad Chrome/Safari; allow microphone.
+
+### DAW
+- In DAW (e.g., Ableton/Logic/Reaper):
+  - Set MIDI input to **ACC Port**.
+  - If using Blue Cat PatchWork, map incoming CCs:
+    - CC7 per channel: Volume
+    - CC80: Solo (toggle)
+    - CC81: Mute (toggle)
+  - Mode B plugin param CCs:
+    - Slot1: CC 20,21,22; Slot2: 30,31,32; ... Slot5: 60,61,62
+
+### OSC
+Default: 127.0.0.1:7000  
+Mode A: `/acc/chords`, `/acc/tempo`  
+Mode B: mirrors fader/solo/mute/transport & plugin params.
+
+## Notes
+- Works without API keys (falls back to simple regex parsing & local UI).
+- OpenAI + Spotify enhance chat/music control if keys are present.
+- MongoDB optional; if not set, Mode A chat persists for current session only.

AudioCommandController/acc/__init__.py

@@ -0,0 +1 @@
+# package marker

AudioCommandController/acc/asr.py

@@ -0,0 +1,23 @@
+import streamlit as st
+from streamlit_webrtc import WebRtcMode, RTCConfiguration, webrtc_streamer
+
+RTC_CFG = RTCConfiguration({"iceServers":[{"urls":["stun:stun.l.google.com:19302"]}]})
+
+def render_mic(title="🎙️ Microphone"):
+    st.markdown(f"### {title}")
+    ctx = webrtc_streamer(
+        key="acc_webrtc",
+        mode=WebRtcMode.SENDONLY,
+        audio=True, video=False,
+        rtc_configuration=RTC_CFG,
+        media_stream_constraints={"audio": True, "video": False}
+    )
+    return ctx
+
+def transcribe_once(ctx) -> str:
+    """
+    Stub transcription. Real Whisper streaming requires audio capture/processing.
+    Use this stub to keep UI flow. You can integrate server-side audio processing later.
+    """
+    # Show a friendly placeholder; this keeps iPad UX ready.
+    return ""

AudioCommandController/acc/db.py

@@ -0,0 +1,32 @@
+import os
+from typing import List, Dict
+try:
+    from pymongo import MongoClient
+except Exception:
+    MongoClient = None
+
+def _get_collection():
+    if not MongoClient:
+        return None
+    uri = os.getenv("MONGO_URI")
+    if not uri:
+        return None
+    client = MongoClient(uri)
+    db = client[os.getenv("MONGO_DB", "acc")]
+    col = db[os.getenv("MONGO_COLLECTION", "mode_a_history")]
+    return col
+
+def save_message(buddy_idx: int, role: str, content: str, session_id: str):
+    col = _get_collection()
+    if not col:
+        return False
+    col.insert_one({"buddy": buddy_idx, "role": role, "content": content, "session": session_id})
+    return True
+
+def fetch_history(buddy_idx: int, session_id: str, limit: int = 100) -> List[Dict]:
+    col = _get_collection()
+    if not col:
+        return []
+    cur = col.find({"buddy": buddy_idx, "session": session_id}).sort("_id", -1).limit(limit)
+    out = list(cur)
+    return list(reversed(out))

AudioCommandController/acc/llm.py

@@ -0,0 +1,32 @@
+import os
+from typing import Optional
+
+try:
+    from openai import OpenAI
+    _client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+except Exception:
+    _client = None
+
+_DEFAULT_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+
+def chat(system: str, user: str, history: Optional[str] = None) -> str:
+    """
+    Simple wrapper around OpenAI; returns a string.
+    Falls back to a naive echo if no API key present.
+    """
+    if not _client:
+        # offline fallback
+        return f"(offline) You said: {user}"
+    messages = [{"role":"system","content":system}]
+    if history:
+        messages.append({"role":"user","content": f"Conversation so far:\n{history}"})
+    messages.append({"role":"user","content": user})
+    try:
+        r = _client.chat.completions.create(
+            model=_DEFAULT_MODEL,
+            messages=messages,
+            temperature=0.6,
+        )
+        return r.choices[0].message.content.strip()
+    except Exception as e:
+        return f"(LLM error) {e}"

AudioCommandController/acc/midi.py

@@ -0,0 +1,77 @@
+import os
+import mido
+from mido import Message
+
+_MIDI_PORT_NAME = os.getenv("MIDI_PORT", "ACC Port")
+
+class MidiOut:
+    def __init__(self):
+        try:
+            self.out = mido.open_output(_MIDI_PORT_NAME)
+            self.ok = True
+            self.msg = f"Opened MIDI port '{_MIDI_PORT_NAME}'"
+        except Exception as e:
+            self.out = None
+            self.ok = False
+            self.msg = f"MIDI port error: {e}"
+
+    def send(self, msg: Message):
+        if self.out:
+            self.out.send(msg)
+
+def send_cc(midi: MidiOut, channel: int, control: int, value: int):
+    if not midi or not midi.out:
+        return False, "MIDI not available"
+    midi.send(Message('control_change', channel=channel, control=control, value=value))
+    return True, "OK"
+
+def send_note(midi: MidiOut, channel: int, note: int, velocity: int=127):
+    if not midi or not midi.out:
+        return False, "MIDI not available"
+    midi.send(Message('note_on', channel=channel, note=note, velocity=velocity))
+    return True, "OK"
+
+def send_pitchbend(midi: MidiOut, channel: int, value14: int):
+    if not midi or not midi.out:
+        return False, "MIDI not available"
+    midi.send(Message('pitchwheel', channel=channel, pitch=value14-8192))
+    return True, "OK"
+
+# Faderport-like mappings
+def send_event(midi: MidiOut, event: str, ch: int, on: bool):
+    if event == "play":
+        return send_note(midi, 0, 93, 127 if on else 0)
+    if event == "stop":
+        return send_note(midi, 0, 94, 127 if on else 0)
+    if event == "record":
+        return send_note(midi, 0, 95, 127 if on else 0)
+    if event == "hui_play":
+        return send_note(midi, 0, 0x5E, 127 if on else 0)
+    if event == "hui_stop":
+        return send_note(midi, 0, 0x5D, 127 if on else 0)
+    if event == "hui_record":
+        return send_note(midi, 0, 0x5F, 127 if on else 0)
+    return False, "Unknown event"
+
+def parse_and_send(midi: MidiOut, text: str):
+    # Basic catch-all; you can extend with more regex
+    if "play" in text.lower():
+        return send_event(midi, "play", 0, True)
+    if "stop" in text.lower():
+        return send_event(midi, "stop", 0, True)
+    if "record" in text.lower() or "rec" in text.lower():
+        return send_event(midi, "record", 0, True)
+    return False, "No known command"
+
+# Mode A helpers
+def send_chords_via_osc(chords: str):
+    from .osc import OscOut
+    o = OscOut()
+    o.configure_from_env()
+    o.send("/acc/chords", chords)
+
+def send_tempo_via_osc(bpm: int):
+    from .osc import OscOut
+    o = OscOut()
+    o.configure_from_env()
+    o.send("/acc/tempo", int(bpm))

AudioCommandController/acc/mode_a.py

@@ -0,0 +1,136 @@
+import streamlit as st
+from typing import List, Dict
+from acc.storage import load_doc_text
+from acc.db import save_message, fetch_history
+from acc.llm import chat
+from acc.suno import generate as suno_generate, extract_stems
+from acc.midi import send_chords_via_osc, send_tempo_via_osc
+from acc.spotify import get_spotify, search_tracks, play_uri
+
+def _init():
+    s = st.session_state
+    if "a_buddy_idx" not in s: s.a_buddy_idx = 0
+    if "a_session" not in s: s.a_session = "session-1"
+    if "a_instr" not in s:
+        s.a_instr = [
+            "Buddy 1 — Music theory & education. Be clear and friendly.",
+            "Buddy 2 — Command Buddy. May output MIDI/OSC on request.",
+            "Buddy 3 — Spotify Buddy. Help find & play reference tracks."
+        ]
+    if "a_docs" not in s: s.a_docs = [[],[],[]]
+    if "a_hist" not in s: s.a_hist = [[],[],[]]  # in-memory if no Mongo
+    if "a_loaded" not in s:
+        # try load from DB (best-effort)
+        for i in range(3):
+            db_msgs = fetch_history(i, s.a_session, 100)
+            s.a_hist[i] = [{"role": m["role"], "content": m["content"]} for m in db_msgs]
+
+def _buddy_tabs():
+    s = st.session_state
+    cols = st.columns(3)
+    for i, c in enumerate(cols):
+        with c:
+            if st.button(f"Buddy {i+1}", key=f"a_b{i}"):
+                s.a_buddy_idx = i
+
+def _sidebar():
+    s = st.session_state
+    st.sidebar.markdown("### Buddy Instructions")
+    txt = st.sidebar.text_area("Edit", value=s.a_instr[s.a_buddy_idx], height=120, key="a_instr_edit")
+    s.a_instr[s.a_buddy_idx] = txt
+
+    st.sidebar.markdown("### Buddy Docs")
+    files = st.sidebar.file_uploader("Upload", type=["pdf","txt","md"], accept_multiple_files=True, key="a_docs_upl")
+    if files:
+        for f in files:
+            text = load_doc_text(f)
+            if text:
+                s.a_docs[s.a_buddy_idx].append({"name": f.name, "text": text})
+        st.sidebar.success(f"Added {len(files)} docs")
+    if st.sidebar.button("Clear docs"):
+        s.a_docs[s.a_buddy_idx] = []
+
+    st.sidebar.markdown("---\n### Suno")
+    suno_prompt = st.sidebar.text_input("Prompt", key="a_suno_p")
+    c1, c2 = st.sidebar.columns(2)
+    with c1:
+        if st.button("Generate", key="a_suno_gen"):
+            ok, msg = suno_generate(suno_prompt); (st.sidebar.success if ok else st.sidebar.error)(msg)
+    with c2:
+        if st.button("Extract Stems", key="a_suno_stems"):
+            ok, msg = extract_stems(suno_prompt); (st.sidebar.success if ok else st.sidebar.error)(msg)
+
+    st.sidebar.markdown("---\n### Music Params → OSC")
+    chords = st.sidebar.text_input("Chord progression", key="a_chords")
+    if st.sidebar.button("Send progression", key="a_chords_send"):
+        send_chords_via_osc(chords); st.sidebar.success("Chords sent")
+    tempo = st.sidebar.number_input("Tempo (BPM)", 40, 240, 120, key="a_tempo")
+    if st.sidebar.button("Send tempo", key="a_tempo_send"):
+        send_tempo_via_osc(int(tempo)); st.sidebar.success("Tempo sent")
+
+def _chat_area():
+    s = st.session_state
+    st.subheader(f"Buddy {s.a_buddy_idx+1}")
+    st.markdown(f"**Instructions:** {s.a_instr[s.a_buddy_idx]}")
+    docs = s.a_docs[s.a_buddy_idx]
+    if docs:
+        with st.expander("Documents", expanded=False):
+            for d in docs:
+                st.write(f"• {d['name']}")
+    st.divider()
+    for m in s.a_hist[s.a_buddy_idx][-100:]:
+        with st.chat_message(m["role"]):
+            st.markdown(m["content"])
+
+    msg = st.text_input("Message", key="a_input")
+    if st.button("Send", key="a_send"):
+        _handle_user(msg)
+
+def _handle_user(text: str):
+    if not text: return
+    s = st.session_state
+    i = s.a_buddy_idx
+    s.a_hist[i].append({"role":"user", "content": text})
+    save_message(i, "user", text, s.a_session)
+
+    # Compose system from instructions + docs
+    docs_text = " ".join(d["text"] for d in s.a_docs[i])[:12000]
+    sys = f"{s.a_instr[i]}\n\nDocs:\n{docs_text}"
+    history_text = "\n".join(f"{m['role']}: {m['content']}" for m in s.a_hist[i][-10:])
+    reply = chat(sys, text, history_text)
+
+    s.a_hist[i].append({"role":"assistant", "content": reply})
+    save_message(i, "assistant", reply, s.a_session)
+    st.experimental_rerun()
+
+def _spotify_section():
+    s = st.session_state
+    if s.a_buddy_idx != 2:
+        return
+    st.divider()
+    st.markdown("### 🎧 Spotify Buddy")
+    sp = get_spotify()
+    if not sp:
+        st.info("Login above to enable Spotify controls.")
+        return
+    q = st.text_input("Search track or artist", key="a_sp_q")
+    if st.button("Search", key="a_sp_do"):
+        res = search_tracks(sp, q, limit=5)
+        items = res.get("tracks", {}).get("items", [])
+        if not items:
+            st.warning("No results.")
+        for it in items:
+            name = it["name"]
+            artist = ", ".join(a["name"] for a in it["artists"])
+            uri = it["uri"]
+            st.write(f"• **{name}** — {artist}")
+            if st.button(f"Play: {name}", key=f"a_play_{it['id']}"):
+                ok, msg = play_uri(sp, uri)
+                (st.success if ok else st.error)(msg)
+
+def render_mode_a():
+    _init()
+    _buddy_tabs()
+    _sidebar()
+    _chat_area()
+    _spotify_section()