From 82759ee536cd8694a1ad9d21808328dfdfe238af Mon Sep 17 00:00:00 2001 From: clement Date: Wed, 17 Jun 2026 14:39:42 +0200 Subject: [PATCH] feat(plip): voice-activated two-phase capture Listen-loop capture now only commits when the caller actually speaks: phase A waits for a sustained voice onset (3 frames above threshold, rejecting the PA-mute click), phase B records until silence. Empty captures are no longer posted, so the NPC never replies to silence. Also: DC-blocking high-pass on the captured mono, VAD thresholds tuned to the quiet SLIC handset mic (onset ~1.4%, silence ~0.6%), keep the PA on during capture (muting it collapsed the mic), and add a /debug/miccap diagnostic endpoint (raw fixed-duration mic capture). --- plip_voice/main/audio.c | 122 +++++++++++++++++++++++---------- plip_voice/main/conversation.c | 14 ++-- plip_voice/main/net.c | 63 ++++++++++++++++- 3 files changed, 154 insertions(+), 45 deletions(-) diff --git a/plip_voice/main/audio.c b/plip_voice/main/audio.c index 4abd3c1..2e97d4c 100644 --- a/plip_voice/main/audio.c +++ b/plip_voice/main/audio.c @@ -603,62 +603,114 @@ int audio_capture_wav(uint8_t *out, size_t out_max, int max_ms, int silence_ms) * speech peaks ~5 % FS / RMS ~1-2 %, noise floor ~0.6 %. Onset must sit * between (catch quiet speech), silence ABOVE the noise floor (so the turn * ends when the caller stops instead of running to max_ms). */ - const int32_t vad_onset_rms_sq = 550 * 550; /* ~1.7% FS² — close-talk speech above room ambient */ - const int32_t vad_silence_rms_sq = 400 * 400; /* ~1.2% FS² — above room ambient so the turn ends */ + /* Measured on the SLIC handset (DC-blocked): speech body sits ~1 % FS RMS + * with peaks ~18 %, noise floor ~0.3 %. The previous silence threshold + * (1.2 %) was ABOVE the speech body, so the recorder classified the + * caller's own voice as silence and cut the turn after ~0.6 s → empty STT. + * Onset must clear noise (3-frame sustained confirm also guards it); silence + * must sit between the noise floor and the speech body so the turn ends only + * on a real pause. */ + const int32_t vad_onset_rms_sq = 450 * 450; /* ~1.4% FS² — speech onset above noise */ + const int32_t vad_silence_rms_sq = 200 * 200; /* ~0.6% FS² — below speech body, above noise floor */ - bool voice_started = false; int silent_frames = 0; int total_frames = 0; + /* One-pole DC blocker (high-pass ~80 Hz) applied to the captured mono. + * The SLIC/handset path carries a huge DC/sub-audio offset (~80 % FS, + * measured). Left in, it (a) swamps the faint voice and (b) keeps the VAD + * RMS permanently above the silence threshold, so the capture never ends + * and always runs the full max_ms — and the gateway then sees only a DC + * transient, transcribing empty. Removing it lets the VAD track the real + * speech envelope (onset/silence work) and hands the gateway a clean, + * voice-dominated signal. Telephone band (300-3400 Hz) is untouched. + * y[n] = x[n] - x[n-1] + R*y[n-1]; R=0.97 → cutoff ~80 Hz at 16 kHz. */ + float dc_x1 = 0.0f, dc_y1 = 0.0f; + const float dc_R = 0.97f; + ESP_LOGI(TAG, "capture: RX enabled, max_ms=%d silence_ms=%d", max_ms, silence_ms); - for (int f = 0; f < max_frames && pcm_written + frame_out_bytes <= pcm_max; f++) { + /* ── Phase A: wait for SUSTAINED voice before committing to a capture ───── + * Only start recording when the caller actually speaks. We require the + * DC-blocked RMS to stay above the onset threshold for ONSET_CONFIRM + * consecutive frames (~60 ms) — a single loud frame is rejected as a + * transient (e.g. the click when the PA mutes just before capture). Audio + * is discarded during this phase. If no sustained voice arrives within + * max_ms we return an empty WAV so the caller posts nothing (the NPC must + * not "reply" to silence). The listen loop simply calls us again. */ + const int ONSET_CONFIRM = 3; + int onset_run = 0; + bool got_onset = false; + for (int f = 0; f < max_frames; f++) { size_t bytes_read = 0; ret = i2s_channel_read(s_mic_handle, rx_buf, frame_in_bytes, &bytes_read, pdMS_TO_TICKS(100)); - if (ret != ESP_OK || bytes_read == 0) { - ESP_LOGW(TAG, "capture: read error f=%d: %s", f, esp_err_to_name(ret)); - continue; - } - - /* Downmix stereo → mono, compute RMS². */ - int16_t *out_frame = (int16_t *)(pcm_out + pcm_written); - int64_t rms_sq = 0; - int n_stereo = (int)(bytes_read / (2 * sizeof(int16_t))); + if (ret != ESP_OK || bytes_read == 0) continue; + int64_t rms_sq = 0; + int n_stereo = (int)(bytes_read / (2 * sizeof(int16_t))); for (int i = 0; i < n_stereo; i++) { - int32_t l = rx_buf[i * 2]; - int32_t r = rx_buf[i * 2 + 1]; - int16_t mono = (int16_t)((l + r) / 2); - out_frame[i] = mono; - rms_sq += (int64_t)mono * mono; + float xf = (float)((rx_buf[i * 2] + rx_buf[i * 2 + 1]) / 2); + float yf = xf - dc_x1 + dc_R * dc_y1; /* DC-blocking high-pass */ + dc_x1 = xf; dc_y1 = yf; + int32_t s = (int32_t)yf; + rms_sq += (int64_t)s * s; } rms_sq /= (n_stereo > 0 ? n_stereo : 1); - - /* VAD logic. */ - if (!voice_started) { - if (rms_sq >= vad_onset_rms_sq) { - voice_started = true; - silent_frames = 0; - ESP_LOGI(TAG, "capture: voice onset at frame %d (rms²=%"PRId64")", f, rms_sq); + if (rms_sq >= vad_onset_rms_sq) { + if (++onset_run >= ONSET_CONFIRM) { + got_onset = true; + ESP_LOGI(TAG, "capture: sustained voice onset at frame %d (rms²=%"PRId64")", + f, rms_sq); + break; } - /* Before voice onset: still accumulate (to avoid clipping onset). - * But don't count towards silence timeout yet. */ } else { + onset_run = 0; + } + } + + if (got_onset) { + /* ── Phase B: record the utterance until silence_ms of silence ──────── */ + for (int f = 0; f < max_frames && pcm_written + frame_out_bytes <= pcm_max; f++) { + size_t bytes_read = 0; + ret = i2s_channel_read(s_mic_handle, rx_buf, frame_in_bytes, + &bytes_read, pdMS_TO_TICKS(100)); + if (ret != ESP_OK || bytes_read == 0) { + ESP_LOGW(TAG, "capture: read error f=%d: %s", f, esp_err_to_name(ret)); + continue; + } + + /* Downmix stereo → mono (DC-blocked), compute RMS². */ + int16_t *out_frame = (int16_t *)(pcm_out + pcm_written); + int64_t rms_sq = 0; + int n_stereo = (int)(bytes_read / (2 * sizeof(int16_t))); + for (int i = 0; i < n_stereo; i++) { + float xf = (float)((rx_buf[i * 2] + rx_buf[i * 2 + 1]) / 2); + float yf = xf - dc_x1 + dc_R * dc_y1; + dc_x1 = xf; dc_y1 = yf; + if (yf > 32767.0f) yf = 32767.0f; + else if (yf < -32768.0f) yf = -32768.0f; + int16_t mono = (int16_t)yf; + out_frame[i] = mono; + rms_sq += (int64_t)mono * mono; + } + rms_sq /= (n_stereo > 0 ? n_stereo : 1); + + pcm_written += (size_t)n_stereo * sizeof(int16_t); + total_frames = f + 1; + if (rms_sq < vad_silence_rms_sq) { - silent_frames++; - if (silent_frames >= silence_frames) { - total_frames = f + 1; - ESP_LOGI(TAG, "capture: VAD end (silence %d frames at f=%d)", silent_frames, f); - pcm_written += (size_t)n_stereo * sizeof(int16_t); + if (++silent_frames >= silence_frames) { + ESP_LOGI(TAG, "capture: VAD end (silence %d frames at f=%d)", + silent_frames, f); break; } } else { silent_frames = 0; } } - - pcm_written += (size_t)n_stereo * sizeof(int16_t); - total_frames = f + 1; + } else { + ESP_LOGI(TAG, "capture: no sustained voice in %d ms — nothing to send", max_ms); + /* pcm_written stays 0 → empty WAV → caller skips posting. */ } /* Leave RX enabled (full-duplex, as at boot). Disabling it here would make diff --git a/plip_voice/main/conversation.c b/plip_voice/main/conversation.c index 661ee36..b6ba1b8 100644 --- a/plip_voice/main/conversation.c +++ b/plip_voice/main/conversation.c @@ -313,17 +313,13 @@ static void conv_task(void *arg) if (!s_offhook) break; vTaskDelay(pdMS_TO_TICKS(200)); /* let the I2S DMA tail drain */ - /* Kill the earpiece amp during capture: at full volume + 24 dB - * mic gain the playback couples into the handset mic at ~50 % - * FS and swamps the caller's voice (capture transcribed empty). - * PA off = no echo path; restored before the reply plays. */ - audio_pa_set(false); - vTaskDelay(pdMS_TO_TICKS(60)); /* PA mute settle */ - - /* Capture caller utterance — earpiece muted, nothing plays. */ + /* Capture the caller utterance with the PA LEFT ON. The loop + * already waits for any playback to finish above, so nothing + * is driving the earpiece during capture — there is no echo to + * mute. Cutting the PA here was found to collapse the mic level + * (captures came back near-silent / DC only), so keep it on. */ int n = audio_capture_wav(cap_buf, cap_max, cap_ms, CAPTURE_SILENCE_MS); - audio_pa_set(true); /* restore for filler/reply */ if (!s_offhook) break; /* hung up during capture */ if (n <= 44) { diff --git a/plip_voice/main/net.c b/plip_voice/main/net.c index 517525b..10fe2e3 100644 --- a/plip_voice/main/net.c +++ b/plip_voice/main/net.c @@ -24,6 +24,7 @@ #include #include "audio.h" +#include "tones.h" #include "cmd_exec.h" #include "phone.h" #include "conversation.h" @@ -600,6 +601,61 @@ static esp_err_t handle_debug_dacvol(httpd_req_t *req) return send_json(req, "200 OK", resp); } +/* ── GET /debug/miccap?ms=4000 (TEMP DIAG: raw fixed-duration mic capture) ─── + * No VAD, DC-blocked, PA on, tones stopped. Lets us measure the true handset + * mic level while the caller speaks continuously, independent of VAD timing. */ +static esp_err_t handle_debug_miccap(httpd_req_t *req) +{ + char query[64] = {0}; + httpd_req_get_url_query_str(req, query, sizeof(query)); + char val[16] = {0}; + int ms = 4000; + if (httpd_query_key_value(query, "ms", val, sizeof(val)) == ESP_OK) { + int v = atoi(val); + if (v > 0 && v <= 8000) ms = v; + } + tones_stop(); + audio_pa_set(true); + if (audio_capture_begin(ms, ms) != 0) { + httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "capture init failed"); + return ESP_FAIL; + } + httpd_resp_set_type(req, "audio/wav"); + const uint32_t cap_rate = 16000; const uint16_t cap_ch = 1, cap_bits = 16; + const uint32_t byte_rate = cap_rate * cap_ch * (cap_bits / 8); + const uint16_t block_align = (uint16_t)(cap_ch * (cap_bits / 8)); + const uint32_t big = 0xFFFFFFFFu; uint16_t fmt_pcm = 1; uint32_t fmt_size = 16; + uint8_t hdr[44]; + memcpy(hdr, "RIFF", 4); memcpy(hdr + 4, &big, 4); + memcpy(hdr + 8, "WAVE", 4); memcpy(hdr + 12, "fmt ", 4); + memcpy(hdr + 16, &fmt_size, 4); memcpy(hdr + 20, &fmt_pcm, 2); + memcpy(hdr + 22, &cap_ch, 2); memcpy(hdr + 24, &cap_rate, 4); + memcpy(hdr + 28, &byte_rate, 4); memcpy(hdr + 32, &block_align, 2); + memcpy(hdr + 34, &cap_bits, 2); memcpy(hdr + 36, "data", 4); memcpy(hdr + 40, &big, 4); + httpd_resp_send_chunk(req, (const char *)hdr, sizeof(hdr)); + + const int max_frames = ms / 20; + int16_t mono[320]; + float dc_x1 = 0.0f, dc_y1 = 0.0f; const float dc_R = 0.97f; + for (int f = 0; f < max_frames; f++) { + int64_t rms_sq = 0; + int n = audio_capture_read_frame(mono, 320, &rms_sq); + if (n < 0) break; + if (n == 0) continue; + for (int i = 0; i < n; i++) { + float xf = (float)mono[i]; + float yf = xf - dc_x1 + dc_R * dc_y1; + dc_x1 = xf; dc_y1 = yf; + if (yf > 32767.0f) yf = 32767.0f; else if (yf < -32768.0f) yf = -32768.0f; + mono[i] = (int16_t)yf; + } + if (httpd_resp_send_chunk(req, (const char *)mono, (ssize_t)(n * sizeof(int16_t))) != ESP_OK) break; + } + audio_capture_end(); + httpd_resp_send_chunk(req, NULL, 0); + return ESP_OK; +} + /* ── GET /debug/getfile?path=/x.wav (read a SPIFFS file back for diagnosis) ── */ static esp_err_t handle_debug_getfile(httpd_req_t *req) @@ -756,7 +812,7 @@ esp_err_t net_init(void) /* Start HTTP server. */ httpd_config_t hcfg = HTTPD_DEFAULT_CONFIG(); hcfg.server_port = 80; - hcfg.max_uri_handlers = 16; + hcfg.max_uri_handlers = 17; hcfg.stack_size = 8192; esp_err_t ret = httpd_start(&s_httpd, &hcfg); @@ -801,6 +857,10 @@ esp_err_t net_init(void) .uri = "/debug/getfile", .method = HTTP_GET, .handler = handle_debug_getfile, .user_ctx = NULL, }; + static const httpd_uri_t uri_debug_miccap = { + .uri = "/debug/miccap", .method = HTTP_GET, + .handler = handle_debug_miccap, .user_ctx = NULL, + }; static const httpd_uri_t uri_debug_dacvol = { .uri = "/debug/dacvol", .method = HTTP_GET, .handler = handle_debug_dacvol, .user_ctx = NULL, @@ -834,6 +894,7 @@ esp_err_t net_init(void) httpd_register_uri_handler(s_httpd, &uri_debug_dial); httpd_register_uri_handler(s_httpd, &uri_debug_vol); httpd_register_uri_handler(s_httpd, &uri_debug_getfile); + httpd_register_uri_handler(s_httpd, &uri_debug_miccap); httpd_register_uri_handler(s_httpd, &uri_debug_dacvol); httpd_register_uri_handler(s_httpd, &uri_debug_ring); httpd_register_uri_handler(s_httpd, &uri_debug_ringstop); -- 2.52.0