Transcribe during recording with recoverable chunks and microphone activity icon

blurt.py

@@ -28,6 +28,9 @@ from PIL import Image, ImageDraw
 from scipy import signal as dsp
 from scipy.io import wavfile
 
+from blurt_client import TranscriptionClient, TranscriptionConfig
+from blurt_session import RecordingSession, RecoveryStore, TranscriptionScheduler
+
 OPENAI_TRANSCRIPTIONS_URL = "https://api.openai.com/v1/audio/transcriptions"
 OPENROUTER_TRANSCRIPTIONS_URL = "https://openrouter.ai/api/v1/audio/transcriptions"
 OPENROUTER_APP_URL = "https://git.felixm.de/felixm/blurt"
@@ -835,6 +838,21 @@ class BlurtApp:
         self.chunks_total = 0
         self.chunks_done = 0
         self._inflight_spool: set[Path] = set()
+        self.streaming = os.environ.get("BLURT_STREAMING", "1") != "0"
+        self._client = TranscriptionClient()
+        self._scheduler = TranscriptionScheduler(self._client.transcribe, workers=CHUNK_WORKERS)
+        self._recovery = RecoveryStore(SPOOL_DIR)
+        self._live_lock = threading.RLock()
+        self._delivery_lock = threading.Lock()
+        self._live_sessions: list[RecordingSession] = []
+        self._recording_session: RecordingSession | None = None
+        self._retrying = False
+        self._retry_thread: threading.Thread | None = None
+        self._quitting = False
+        self._meter_stop = threading.Event()
+        self._meter_frame = None
+        self._meter_level = 0.0
+        self._meter_icons = [self._create_recording_image(i / 11) for i in range(12)]
         spooled = self._spooled_files()
         if spooled:
             logging.warning(
@@ -914,6 +932,16 @@ class BlurtApp:
             if self.recording:
                 color = (220, 40, 40)  # red
                 state = "Recording..."
+                active = self._recording_session
+                if active is not None:
+                    state = f"Recording · {active.done}/{active.total} chunks processed"
+            elif self._live_sessions:
+                color = (40, 100, 220)
+                sessions = tuple(self._live_sessions)
+                done = sum(s.done for s in sessions)
+                total = sum(s.total for s in sessions)
+                state = f"Finishing {done}/{total} chunks..."
+                progress = done / total if total else None
             elif self.pending_transcriptions > 0:
                 color = (40, 100, 220)  # blue
                 if total > 1:
@@ -924,15 +952,79 @@ class BlurtApp:
             else:
                 color = (120, 120, 120)  # gray
                 state = "Ready"
-            self.icon.icon = self._create_icon_image(color, progress)
+            self.icon.icon = (
+                self._meter_icons[round(self._meter_level * 11)]
+                if self.recording else self._create_icon_image(color, progress)
+            )
             self.icon.title = f"{APP_NAME} - {state}"
             self.icon.update_menu()
 
     def _audio_callback(self, indata, frames, time_info, status):
+        if hasattr(self, "_meter_stop"):
+            # Keep only a small, owned snapshot. Level calculation and tray
+            # drawing happen outside PortAudio's callback.
+            self._meter_frame = (
+                time.monotonic(), indata[::max(1, frames // 512)].copy(),
+            )
+        active = self._recording_session
+        if active is not None:
+            if status:
+                active.report_input_status(str(status))
+            active.feed(indata)
+            return
         if status:
             logging.warning("sounddevice input status: %s", status)
         self.audio_frames.append(indata.copy())
 
+    def _create_recording_image(self, level: float) -> Image.Image:
+        color = (round(160 + 80 * level), round(28 + 32 * level), 40)
+        img = self._create_icon_image(color)
+        draw = ImageDraw.Draw(img)
+        for x, scale in ((21, 0.6), (30, 1.0), (39, 0.75)):
+            height = round(4 + 30 * level * scale)
+            draw.rounded_rectangle(
+                (x, 32 - height // 2, x + 4, 32 + height // 2),
+                radius=2, fill=(255, 235, 235),
+            )
+        return img
+
+    @staticmethod
+    def _microphone_level(samples: np.ndarray) -> float:
+        mono = samples.mean(axis=1) if samples.ndim == 2 else samples
+        if not len(mono):
+            return 0.0
+        centered = mono - mono.mean()
+        rms = float(np.sqrt(np.mean(centered * centered)))
+        dbfs = 20 * np.log10(max(rms, 1e-6))
+        return float(np.clip((dbfs + 60) / 42, 0, 1))
+
+    def _animate_microphone(self):
+        previous = None
+        while not self._meter_stop.wait(0.1):
+            snapshot = self._meter_frame
+            target = 0.0
+            if self.recording and snapshot is not None:
+                captured_at, samples = snapshot
+                if time.monotonic() - captured_at < 0.3:
+                    target = self._microphone_level(samples)
+            # Fast attack, gentle release: speech is obvious without strobing.
+            self._meter_level = max(target, self._meter_level * 0.65) if self.recording else 0.0
+            step = round(self._meter_level * 11)
+            if not self.recording:
+                previous = None
+                continue
+            if step == previous:
+                continue
+            try:
+                with self._icon_lock:
+                    if self.recording and self.icon and not self._quitting:
+                        self.icon.icon = self._meter_icons[step]
+                        previous = step
+                # Deliberately leave menu/title updates to state transitions.
+            except Exception:
+                logging.exception("Could not update microphone activity icon")
+                return
+
     def _provider_for_model(self, model: str) -> str:
         for model_id, _, provider in TRANSCRIPTION_MODELS:
             if model_id == model:
@@ -992,6 +1084,8 @@ class BlurtApp:
     def _start_recording(self):
         if not self._require_api_key("Recording"):
             return
+        self._meter_frame = None
+        self._meter_level = 0.0
         self.audio_frames = []
         try:
             self.capture_rate, self.capture_channels = input_stream_settings(self.device_index)
@@ -1406,10 +1500,71 @@ class BlurtApp:
         return [path for path in self._spooled_files() if path not in inflight]
 
     def _retry_spooled(self, icon=None, item=None):
+        if self.streaming:
+            with self._live_lock:
+                if self._retrying or self._quitting:
+                    return
+                self._retrying = True
+            self._retry_thread = threading.Thread(target=self._retry_sessions, daemon=True, name="RetrySessions")
+            self._retry_thread.start()
+            return
         if not self._require_api_key("Retry"):
             return
         threading.Thread(target=self._retry_spooled_worker, daemon=True, name="RetrySpool").start()
 
+    def _retry_count(self):
+        if self.streaming:
+            return len(self._recovery.discover()) + len(self._retryable_files())
+        return len(self._retryable_files())
+
+    def _retry_sessions(self):
+        collected = []
+        recovered = []
+        attempted = []
+        try:
+            entries = [(r.created_at, r, None) for r in self._recovery.discover()]
+            entries.extend((p.stat().st_mtime, None, p) for p in self._retryable_files())
+            for _, record, path in sorted(entries, key=lambda entry: entry[0]):
+                if self._quitting:
+                    break
+                try:
+                    config = self._transcription_config(record.config if record else None)
+                    if not config.api_key:
+                        self.platform.notify("Error", f"Missing {config.provider} key for saved recording.")
+                        continue
+                    if record is not None:
+                        active = self._recovery.resume(record, config, self._scheduler)
+                    else:
+                        active = RecordingSession.from_wav(path, config, SPOOL_DIR, self._scheduler)
+                    attempted.append(active)
+                    result = active.join()
+                    if result and result.text:
+                        collected.append(result.text)
+                        recovered.append(active)
+                    elif result and result.complete:
+                        active.acknowledge_delivery()  # recovered silence needs no clipboard
+                except Exception:
+                    logging.exception("Could not recover recording %s", record or path)
+            if collected and not self._quitting:
+                with self._delivery_lock:
+                    pyperclip.copy("\n\n".join(collected))
+                    for active in recovered:
+                        active.acknowledge_delivery()
+                self.platform.notify("Copied", f"Recovered {len(collected)} recording(s).")
+            elif not self._quitting:
+                self.platform.notify("Retry", "No transcripts recovered; audio remains saved.")
+        except Exception:
+            logging.exception("Session retry failed; recovery data retained")
+            if not self._quitting:
+                self.platform.notify("Error", "Retry failed; audio remains saved.")
+        finally:
+            for active in attempted:
+                active.release_for_retry()
+            with self._live_lock:
+                self._retrying = False
+            if not self._quitting:
+                self._update_icon()
+
     def _retry_spooled_worker(self):
         """Re-upload spooled clips one at a time: they failed once already, and a
         burst of parallel requests is the least likely thing to fix that. Their
@@ -1494,6 +1649,9 @@ class BlurtApp:
             logging.error("Transcription error response: %s", response.text[:1000])
 
     def toggle_recording(self):
+        if self.streaming:
+            self._toggle_streaming()
+            return
         with self.lock:
             if not self.recording:
                 self._start_recording()
@@ -1514,6 +1672,128 @@ class BlurtApp:
                 else:
                     self.platform.notify("Error", "No audio recorded.")
 
+    def _transcription_config(self, metadata=None):
+        metadata = metadata or {}
+        model = metadata.get("model", self.transcription_model)
+        provider = metadata.get("provider", self._provider_for_model(model))
+        return TranscriptionConfig(
+            model=model,
+            language=metadata.get("language", self.transcription_language),
+            provider=provider,
+            api_key=self.api_keys.get(provider, ""),
+        )
+
+    def _session_update(self, session):
+        if not self._quitting:
+            if (session is self._recording_session and session.result is not None
+                    and session.result.errors and not getattr(session, "capture_warned", False)):
+                session.capture_warned = True
+                self.platform.notify("Capture interrupted", "Stop recording; available audio is kept for retry.")
+            self._update_icon()
+
+    def _session_complete(self, session):
+        # Sessions may finish out of order. Deliver in recording order, and keep
+        # delivery outside the microphone/start-stop lock.
+        with self._delivery_lock:
+            while not self._quitting:
+                with self._live_lock:
+                    if not self._live_sessions or self._live_sessions[0].result is None:
+                        break
+                    if self._live_sessions[0] is self._recording_session:
+                        # An input failure can finish the producer before the
+                        # user stops the microphone. Never deliver mid-recording.
+                        break
+                    ready = self._live_sessions.pop(0)
+                result = ready.result
+                try:
+                    if result.text:
+                        if result.complete:
+                            self._deliver(result.text)
+                        else:
+                            pyperclip.copy(result.text)
+                            self.platform.notify("Partial", "Copied with gaps; audio kept for retry.")
+                        ready.acknowledge_delivery()
+                        stopped_at = getattr(ready, "stopped_at", None)
+                        if stopped_at is not None:
+                            logging.info(
+                                "Session delivered: id=%s stop_to_delivery=%.3fs complete=%s",
+                                result.session_id, time.monotonic() - stopped_at, result.complete,
+                            )
+                    elif result.errors:
+                        ready.release_for_retry()
+                        self.platform.notify("Error", "Transcription failed; audio kept for retry.")
+                    else:
+                        ready.acknowledge_delivery()
+                        self.platform.notify("Recording", "No speech detected.")
+                except Exception:
+                    logging.exception("Session delivery failed; recovery data retained")
+                    ready.release_for_retry()
+                    self.platform.notify("Error", "Could not deliver transcript; kept for retry.")
+        self._session_update(session)
+
+    def _toggle_streaming(self):
+        with self.lock:
+            if self._quitting:
+                return
+            if self.recording:
+                self._stop_streaming()
+                return
+            if not self._require_api_key("Recording"):
+                return
+            self._meter_frame = None
+            self._meter_level = 0.0
+            active = None
+            try:
+                self.capture_rate, self.capture_channels = input_stream_settings(self.device_index)
+                active = RecordingSession(
+                    self.capture_rate, self.capture_channels,
+                    self._transcription_config(), SPOOL_DIR, self._scheduler,
+                    on_update=self._session_update, on_complete=self._session_complete,
+                )
+                with self._live_lock:
+                    self._live_sessions.append(active)
+                self._recording_session = active
+                self.stream = sd.InputStream(
+                    device=self.device_index, samplerate=self.capture_rate,
+                    channels=self.capture_channels, dtype=DTYPE,
+                    callback=self._audio_callback,
+                )
+                self.stream.start()
+                self.recording = True
+                logging.info(
+                    "Streaming capture started: session=%s device=%s rate=%d channels=%d",
+                    active.session_id, self.device_name or "<system default>",
+                    self.capture_rate, self.capture_channels,
+                )
+            except Exception as error:
+                logging.exception("Could not start streaming capture")
+                if self.stream is not None:
+                    self.stream.close()
+                    self.stream = None
+                self._recording_session = None
+                if active is not None:
+                    active.stop()
+                self.platform.notify("Error", f"Could not start recording: {error}")
+                return
+            self._update_icon()
+            self.platform.notify("Recording", "Transcribing as you speak. Press hotkey to stop.")
+
+    def _stop_streaming(self):
+        # PortAudio must finish its last callback before the session receives EOF.
+        if self.stream is not None:
+            self.stream.stop()
+            self.stream.close()
+            self.stream = None
+        active = self._recording_session
+        self._recording_session = None
+        self.recording = False
+        if active is not None:
+            active.stopped_at = time.monotonic()
+            active.stop()
+            if active.result is not None:
+                self._session_complete(active)
+        self._update_icon()
+
     def _on_click(self, icon, item):
         self.toggle_recording()
 
@@ -1659,9 +1939,9 @@ class BlurtApp:
             pystray.MenuItem(lambda item: self._api_key_label(), None, enabled=False),
             pystray.MenuItem("Language", pystray.Menu(*language_items)),
             pystray.MenuItem(
-                lambda item: f"Retry failed transcriptions ({len(self._retryable_files())})",
+                lambda item: f"Retry failed transcriptions ({self._retry_count()})",
                 self._retry_spooled,
-                enabled=lambda item: len(self._retryable_files()) > 0,
+                enabled=lambda item: not self._retrying and self._retry_count() > 0,
             ),
             pystray.MenuItem("Show Logs", self._show_logs),
             *awake_items,
@@ -1670,15 +1950,36 @@ class BlurtApp:
         )
 
     def _on_quit(self, icon, item):
+        self._quitting = True
+        self._meter_stop.set()
         if self.recording:
-            self._stop_recording()
+            if self.streaming:
+                self._stop_streaming()
+            else:
+                self._stop_recording()
         if self._awake_keeper and self._awake_keeper.is_active:
             self._awake_keeper.stop()
         self.platform.cleanup_toggle_trigger()
+        # A non-daemon coordinator keeps Python alive after the tray event loop
+        # exits, until capture checkpoints and pending responses are durable.
+        threading.Thread(target=self._finish_shutdown, name="BlurtShutdown", daemon=False).start()
         icon.stop()
 
+    def _finish_shutdown(self):
+        if self.streaming:
+            # Finish checkpointing and uploads without delivering into an app
+            # the user has closed. Undelivered text remains recoverable.
+            for active in tuple(self._live_sessions):
+                active.join()
+                active.release_for_retry()
+            if self._retry_thread is not None:
+                self._retry_thread.join()
+        self._scheduler.shutdown(wait=True)
+        self._client.close()
+
     def _setup(self, icon):
         icon.visible = True
+        threading.Thread(target=self._animate_microphone, name="MicrophoneMeter", daemon=True).start()
 
     def run(self):
         self.platform.setup_toggle_trigger(self)