From 1078040771e869371e14f3569875eded94d3287b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:25:31 +0100 Subject: [PATCH 1/2] feat: add Studio Sound microphone enhancement --- Cargo.lock | 47 +++ apps/desktop-gpui/Cargo.lock | 47 +++ apps/desktop-gpui/src/editor_sidebar.rs | 7 + apps/desktop-gpui/src/editor_tabs.rs | 50 +++ apps/desktop-gpui/src/recording.rs | 36 ++ apps/desktop/src-tauri/src/recording.rs | 13 + .../src/routes/editor/ConfigSidebar.tsx | 52 ++- apps/desktop/src/store.ts | 4 + crates/audio/Cargo.toml | 1 + crates/audio/src/lib.rs | 2 + crates/audio/src/voice.rs | 316 ++++++++++++++++++ crates/editor/src/audio.rs | 177 +++++++++- crates/editor/src/audio_output.rs | 178 ++++++++-- .../editor/src/audio_output/native_tests.rs | 33 ++ crates/editor/src/editor_instance.rs | 10 + crates/editor/src/export_audio.rs | 175 +++++++++- crates/editor/src/playback.rs | 89 ++++- .../src/preparing_preview/native_tests.rs | 145 ++++++++ crates/editor/src/segments.rs | 1 + 19 files changed, 1316 insertions(+), 67 deletions(-) create mode 100644 crates/audio/src/voice.rs diff --git a/Cargo.lock b/Cargo.lock index 9197d09384..280a91700a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,6 +387,12 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "apple-native-keyring-store" version = "1.0.1" @@ -439,6 +445,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -1272,6 +1284,7 @@ dependencies = [ "cidre", "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", + "nnnoiseless", "serde", "serde_json", "tempfile", @@ -3433,6 +3446,19 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + [[package]] name = "either" version = "1.15.0" @@ -4139,6 +4165,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic_singleton" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6e923c8e978e57cf63e2e200ca967d1d20f0ea2662b28f6d4e11c44aa6ab16" +dependencies = [ + "anymap3", + "parking_lot", +] + [[package]] name = "gethostname" version = "1.0.2" @@ -6379,6 +6415,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "easyfft", + "once_cell", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -6473,6 +6519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", + "serde", ] [[package]] diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index 686399628c..11c3982dac 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -453,6 +453,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -479,6 +485,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -1400,6 +1412,7 @@ dependencies = [ "cidre", "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", "ffmpeg-next", + "nnnoiseless", "serde", "serde_json", "tokio", @@ -3408,6 +3421,19 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + [[package]] name = "either" version = "1.17.0" @@ -4124,6 +4150,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic_singleton" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6e923c8e978e57cf63e2e200ca967d1d20f0ea2662b28f6d4e11c44aa6ab16" +dependencies = [ + "anymap3", + "parking_lot", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -6680,6 +6716,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "easyfft", + "once_cell", +] + [[package]] name = "no_std_io2" version = "0.9.4" @@ -6801,6 +6847,7 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "bytemuck", "num-traits", + "serde", ] [[package]] diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index 64bdcfee20..e13dfb0bb5 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -769,6 +769,8 @@ pub enum ColorPickerDrag { /// The sidebar's own state -- everything `ConfigSidebar`'s signals hold that is /// not in the project config. pub struct SidebarState { + pub audio_enhancement_default: bool, + pub audio_enhancement_error: Option, pub(crate) style_target: Option<(usize, StyleGroup)>, pub(crate) image_import_error: Option, pub(crate) image_asset_status: Option<(String, bool)>, @@ -877,6 +879,11 @@ pub struct SidebarState { impl SidebarState { pub fn new(config: &ProjectConfiguration) -> Self { Self { + audio_enhancement_default: crate::store::store_section("audio_enhancement") + .get("enabledByDefault") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false), + audio_enhancement_error: None, style_target: None, image_import_error: None, image_asset_status: None, diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index 1b0c606da2..5a0b24829f 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -1541,6 +1541,7 @@ impl EditorWindow { pub(crate) fn render_audio_tab(&self, cx: &mut Context) -> AnyElement { let theme = self.theme; let audio = &self.project.audio; + let enabled_by_default = self.sidebar.audio_enhancement_default; let summary = self.summary(); let muted = audio.mute; let has_microphone = summary.is_some_and(|summary| summary.has_microphone); @@ -1586,6 +1587,55 @@ impl EditorWindow { })), ), ) + .children(has_microphone.then(|| { + div() + .flex() + .flex_col() + .gap(px(8.)) + .child(ui::Subfield::plain(&theme, "Studio Sound").child( + ui::Toggle::plain(&theme, "audio-improve", audio.improve).on_click( + cx.listener(|this, _, window, cx| { + this.edit_project("audio-improve", window, cx, |project| { + project.audio.improve = !project.audio.improve; + true + }); + }), + ), + )) + .child( + div() + .text_size(px(12.)) + .text_color(theme.gray_10) + .child("Reduce background noise and bring your voice into focus."), + ) + .into_any_element() + })) + .child( + ui::Subfield::plain(&theme, "Studio Sound for new recordings").child( + ui::Toggle::plain(&theme, "audio-improve-default", enabled_by_default) + .on_click(cx.listener(move |this, _, _, cx| { + if !store::set_store_setting( + "audio_enhancement", + "enabledByDefault", + serde_json::json!(!enabled_by_default), + ) { + this.sidebar.audio_enhancement_error = + Some("Could not save the Studio Sound default".into()); + } else { + this.sidebar.audio_enhancement_default = !enabled_by_default; + this.sidebar.audio_enhancement_error = None; + } + cx.notify(); + })), + ), + ) + .children(self.sidebar.audio_enhancement_error.as_ref().map(|error| { + div() + .text_size(px(12.)) + .text_color(theme.gray_10) + .child(error.clone()) + .into_any_element() + })) .children(has_microphone.then(|| { self.slider_field_disabled( "Microphone Volume", diff --git a/apps/desktop-gpui/src/recording.rs b/apps/desktop-gpui/src/recording.rs index 120d7e9ff5..80ef647244 100644 --- a/apps/desktop-gpui/src/recording.rs +++ b/apps/desktop-gpui/src/recording.rs @@ -460,6 +460,14 @@ async fn finalize_studio( )) .unwrap_or_default(); apply_animated_gradient_to_project_config(&project_path, &capture_target, &library); + if crate::store::store_section("audio_enhancement") + .get("enabledByDefault") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + && let Err(error) = enable_studio_sound(&project_path) + { + tracing::warn!(%error, "Could not apply the Studio Sound default"); + } }) .await .context("studio post-finalize task")?; @@ -1461,6 +1469,34 @@ fn blur_mode_json(blur: crate::store::BlurMode) -> &'static str { } } +fn enable_studio_sound(project_path: &std::path::Path) -> std::io::Result<()> { + let path = project_path.join("project-config.json"); + let mut config: serde_json::Value = serde_json::from_slice(&std::fs::read(&path)?)?; + let object = config.as_object_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Project configuration is not an object", + ) + })?; + let audio = object + .entry("audio") + .or_insert_with(|| serde_json::json!({})); + let audio = audio.as_object_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Audio configuration is not an object", + ) + })?; + audio.insert("improve".into(), serde_json::Value::Bool(true)); + let temp = path.with_extension(format!("studio-sound-{}.tmp", crate::store::new_uuid_v4())); + let result = std::fs::write(&temp, serde_json::to_vec_pretty(&config)?) + .and_then(|()| std::fs::rename(&temp, path)); + if result.is_err() { + let _ = std::fs::remove_file(temp); + } + result +} + fn apply_animated_gradient_to_project_config( project_dir: &std::path::Path, capture_target: &ScreenCaptureTarget, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 2123769b9b..8096b61c3e 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6407,6 +6407,19 @@ fn project_config_from_recording( let using_default_config = default_config.is_none(); let mut config = default_config.unwrap_or_default(); + if app + .store("store") + .ok() + .and_then(|store| store.get("audio_enhancement")) + .and_then(|value| { + value + .get("enabledByDefault") + .and_then(serde_json::Value::as_bool) + }) + .unwrap_or(false) + { + config.audio.improve = true; + } if using_default_config { let library = app .store("store") diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 8bed968d9d..a95b1977b2 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -46,7 +46,11 @@ import { createStore, produce } from "solid-js/store"; import { Dynamic } from "solid-js/web"; import toast from "solid-toast"; import { Toggle } from "~/components/Toggle"; -import { animatedGradientsStore, generalSettingsStore } from "~/store"; +import { + animatedGradientsStore, + audioEnhancementStore, + generalSettingsStore, +} from "~/store"; import { listSystemFonts } from "~/utils/fonts"; import { normalizeOpaqueHexColor } from "~/utils/hex-color"; import { @@ -520,6 +524,8 @@ function ConfigSidebarContent() { editorState, meta, } = useEditorContext(); + const audioEnhancement = audioEnhancementStore.createQuery(); + const [savingAudioDefault, setSavingAudioDefault] = createSignal(false); const organizationSelection = createSelectedOrganization(); const brandColorSwatches = createMemo(() => getOrganizationBrandColorSwatches( @@ -791,18 +797,38 @@ function ConfigSidebarContent() { )} - {/* - setProject("audio", "mute", v)} - /> - */} - - {/* - - - - */} + + + + setProject("audio", "improve", enabled) + } + /> + +

+ Reduce background noise and bring your voice into focus. +

+ + + { + setSavingAudioDefault(true); + try { + await audioEnhancementStore.set({ + enabledByDefault: enabled, + }); + await audioEnhancement.refetch(); + } catch { + toast.error("Could not save the Studio Sound default"); + } finally { + setSavingAudioDefault(false); + } + }} + /> + {meta().hasMicrophone && ( (name: string, defaults?: T) { }; } +export const audioEnhancementStore = declareStore<{ + enabledByDefault: boolean; +}>("audio_enhancement", { enabledByDefault: false }); + export const presetsStore = declareStore("presets"); const animatedGradientDefaults: AnimatedGradientLibrary = { presets: [], diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index e0aa1b7ec4..04154e62e0 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" test-support = [] [dependencies] +nnnoiseless = { version = "0.5.2", default-features = false } cap-enc-ffmpeg = { path = "../enc-ffmpeg" } ffmpeg = { workspace = true } cpal = { workspace = true } diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index f267d790a1..da87e13ee8 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -6,6 +6,7 @@ mod renderer; mod streaming; mod sync_analysis; mod transcription_timing; +mod voice; pub use audio_data::*; pub use calibration_store::*; @@ -15,6 +16,7 @@ pub use renderer::*; pub use streaming::*; pub use sync_analysis::*; pub use transcription_timing::*; +pub use voice::*; pub trait FromSampleBytes: cpal::SizedSample + std::fmt::Debug + Send + 'static { const BYTE_SIZE: usize; diff --git a/crates/audio/src/voice.rs b/crates/audio/src/voice.rs new file mode 100644 index 0000000000..89a0c5beda --- /dev/null +++ b/crates/audio/src/voice.rs @@ -0,0 +1,316 @@ +use crate::AudioSampleSource; +use nnnoiseless::DenoiseState; +use std::ops::Range; + +const FRAME: usize = DenoiseState::FRAME_SIZE; +pub const VOICE_PREROLL_SAMPLES: usize = 9_600; +pub const VOICE_WINDOW_PADDING_SAMPLES: usize = FRAME * 3; + +pub struct VoiceEnhancer { + states: Vec>>, + next_input: usize, + expected_start: Option, + frame_start: Option, + frame: Vec, + previous: Vec, +} + +impl VoiceEnhancer { + pub fn new(channels: u16) -> Self { + let channels = usize::from(channels.clamp(1, 2)); + Self { + states: (0..channels).map(|_| DenoiseState::new()).collect(), + next_input: 0, + expected_start: None, + frame_start: None, + frame: vec![0.0; FRAME * channels], + previous: vec![0.0; FRAME * channels], + } + } + + pub fn is_contiguous(&self, start: usize) -> bool { + self.expected_start == Some(start) + || self + .frame_start + .is_some_and(|first| (first..first + FRAME).contains(&start)) + } + + pub fn source_range(&self, start: usize, count: usize) -> Range { + if count == 0 { + return start..start; + } + let first = if self.is_contiguous(start) { + self.next_input + } else { + (start / FRAME * FRAME).saturating_sub(VOICE_PREROLL_SAMPLES) + }; + let end = start + .saturating_add(count) + .div_ceil(FRAME) + .saturating_add(1) + .saturating_mul(FRAME); + first..end.max(first) + } + + pub fn render( + &mut self, + source: &T, + start: usize, + count: usize, + ) -> VoiceAudio { + let channels = self.states.len(); + let count = count.min(source.sample_count().saturating_sub(start)); + if count > 0 && !self.is_contiguous(start) { + self.next_input = self.source_range(start, count).start; + self.states = (0..channels).map(|_| DenoiseState::new()).collect(); + self.frame_start = None; + self.previous.fill(0.0); + } + let mut samples = vec![0.0; count * channels]; + let mut written = 0; + while written < count { + let cursor = start + written; + while !self + .frame_start + .is_some_and(|first| (first..first + FRAME).contains(&cursor)) + { + self.process_frame(source); + } + let offset = cursor - self.frame_start.unwrap(); + let take = (FRAME - offset).min(count - written); + samples[written * channels..(written + take) * channels] + .copy_from_slice(&self.frame[offset * channels..(offset + take) * channels]); + written += take; + } + self.expected_start = Some(start + count); + VoiceAudio { + samples, + channels: channels as u16, + start, + total_samples: source.sample_count(), + } + } + + fn process_frame(&mut self, source: &T) { + let channels = self.states.len(); + let mut input = [0.0; FRAME]; + let mut output = [0.0; FRAME]; + for (channel, state) in self.states.iter_mut().enumerate() { + for (index, sample) in input.iter_mut().enumerate() { + let value = source + .sample((self.next_input + index) * channels + channel) + .copied() + .unwrap_or(0.0); + *sample = if value.is_finite() { + value.clamp(-1.0, 1.0) * 32_768.0 + } else { + 0.0 + }; + } + state.process_frame(&mut output, &input); + for (index, (&clean, &raw)) in output.iter().zip(&input).enumerate() { + let index = index * channels + channel; + self.frame[index] = + (clean / 32_768.0 * 0.9 + self.previous[index] * 0.1).clamp(-1.0, 1.0); + self.previous[index] = raw / 32_768.0; + } + } + // RNNoise emits the preceding 10 ms frame. Reading ahead keeps source timestamps intact. + self.frame_start = self.next_input.checked_sub(FRAME); + self.next_input += FRAME; + } +} + +pub struct VoiceAudio { + samples: Vec, + channels: u16, + start: usize, + total_samples: usize, +} + +impl AudioSampleSource for VoiceAudio { + fn channels(&self) -> u16 { + self.channels + } + fn sample_count(&self) -> usize { + self.total_samples + } + fn sample(&self, index: usize) -> Option<&f32> { + index + .checked_sub(self.start * usize::from(self.channels)) + .and_then(|index| self.samples.get(index)) + } + fn sample_slice(&self, range: Range) -> Option<&[f32]> { + let offset = self.start * usize::from(self.channels); + self.samples + .get(range.start.checked_sub(offset)?..range.end.checked_sub(offset)?) + } +} + +pub enum VoiceSource<'a, T> { + Original(&'a T), + Enhanced(&'a VoiceAudio), +} + +impl AudioSampleSource for VoiceSource<'_, T> { + fn channels(&self) -> u16 { + match self { + Self::Original(source) => source.channels(), + Self::Enhanced(source) => source.channels(), + } + } + fn sample_count(&self) -> usize { + match self { + Self::Original(source) => source.sample_count(), + Self::Enhanced(source) => source.sample_count(), + } + } + fn sample(&self, index: usize) -> Option<&f32> { + match self { + Self::Original(source) => source.sample(index), + Self::Enhanced(source) => source.sample(index), + } + } + fn sample_slice(&self, range: Range) -> Option<&[f32]> { + match self { + Self::Original(source) => source.sample_slice(range), + Self::Enhanced(source) => source.sample_slice(range), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Samples { + data: Vec, + channels: u16, + } + impl AudioSampleSource for Samples { + fn channels(&self) -> u16 { + self.channels + } + fn sample_count(&self) -> usize { + self.data.len() / usize::from(self.channels) + } + fn sample(&self, index: usize) -> Option<&f32> { + self.data.get(index) + } + } + + fn signal(frames: usize, channels: u16) -> Samples { + let mut seed = 42_u32; + Samples { + channels, + data: (0..frames * usize::from(channels)) + .map(|_| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (seed as f64 / u32::MAX as f64 * 0.16 - 0.08) as f32 + }) + .collect(), + } + } + + #[test] + fn arbitrary_blocks_match_whole_processing_including_stereo_and_tail() { + for channels in [1, 2] { + let source = signal(20_137, channels); + let expected = VoiceEnhancer::new(channels).render(&source, 0, source.sample_count()); + let mut enhancer = VoiceEnhancer::new(channels); + let mut actual = Vec::new(); + let mut cursor = 0; + for count in [1, 479, 481, 4_096, 7, 11_999, 3_074] { + let audio = enhancer.render(&source, cursor, count); + cursor += audio.samples.len() / usize::from(channels); + actual.extend(audio.samples); + } + assert_eq!(cursor, source.sample_count()); + assert_eq!(actual, expected.samples); + assert!(enhancer.render(&source, cursor, 512).samples.is_empty()); + } + } + + #[test] + fn seeks_prime_from_a_bounded_source_window() { + let source = signal(120_001, 1); + let mut enhancer = VoiceEnhancer::new(1); + enhancer.render(&source, 0, 4_096); + for start in [96_177, 13, 60_050] { + let range = enhancer.source_range(start, 4096); + assert!(range.len() <= 4096 + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES); + let actual = enhancer.render(&source, start, 4096); + let expected = VoiceEnhancer::new(1).render(&source, start, 4096); + assert_eq!(actual.samples, expected.samples); + } + } + + #[test] + fn every_frame_alignment_fits_the_bounded_source_window() { + let enhancer = VoiceEnhancer::new(1); + for remainder in 0..FRAME { + let range = enhancer.source_range(48_000 + remainder, 4_096); + assert!(range.len() <= 4_096 + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES); + } + } + + #[test] + fn silence_short_tracks_and_non_finite_samples_stay_safe() { + for frames in [0, 1, 479, 480, 481, 997] { + let source = Samples { + data: vec![0.0; frames], + channels: 1, + }; + let output = VoiceEnhancer::new(1).render(&source, 0, frames + 480); + assert_eq!(output.samples, source.data); + } + let source = Samples { + data: vec![f32::NAN, f32::INFINITY, f32::NEG_INFINITY], + channels: 1, + }; + let output = VoiceEnhancer::new(1).render(&source, 0, 3); + assert_eq!(output.samples, vec![0.0; 3]); + } + + #[test] + fn lookahead_compensates_the_filter_delay() { + let mut source = Samples { + data: vec![0.0; 12_017], + channels: 1, + }; + source.data[5_333] = 0.8; + let output = VoiceEnhancer::new(1).render(&source, 0, source.sample_count()); + let peak = output + .samples + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.abs().total_cmp(&b.abs())) + .unwrap() + .0; + assert_eq!(peak, 5_333); + assert_eq!(output.samples.len(), source.data.len()); + } + + #[test] + fn fan_noise_is_reduced_without_clipping_or_accumulated_buffers() { + let mut source = signal(96_000, 1); + let mut previous = 0.0; + for sample in &mut source.data { + previous = previous * 0.95 + *sample * 0.05; + *sample = previous; + } + let mut enhancer = VoiceEnhancer::new(1); + let output = enhancer.render(&source, 0, source.sample_count()); + let power = |samples: &[f32]| samples.iter().map(|sample| sample * sample).sum::(); + assert!(power(&output.samples[48_000..]) < power(&source.data[48_000..]) * 0.1); + assert!( + output + .samples + .iter() + .all(|sample| sample.is_finite() && sample.abs() <= 1.0) + ); + assert_eq!(enhancer.states.len(), 1); + assert_eq!(enhancer.frame.len(), FRAME); + assert_eq!(enhancer.previous.len(), FRAME); + } +} diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 8f5c8c3545..942727523c 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -1,7 +1,7 @@ use crate::export_audio::{EXPORT_AUDIO_BLOCK_SAMPLES, ExportAudioError, ExportAudioSources}; use cap_audio::{ - AudioData, AudioRendererTrack, DecodedAudio, FromSampleBytes, StereoMode, - cast_bytes_to_f32_slice, cast_f32_slice_to_bytes, + AudioData, AudioRendererTrack, DecodedAudio, FromSampleBytes, StereoMode, VoiceAudio, + VoiceEnhancer, VoiceSource, cast_bytes_to_f32_slice, cast_f32_slice_to_bytes, }; use cap_media::MediaError; use cap_media_info::AudioInfo; @@ -42,6 +42,7 @@ pub struct AudioRenderer { transition_incoming: Vec, speed_audio_processors: [Option; 2], speed_audio_use_counter: u64, + voice_enhancement: VoiceEnhancementCache, } #[derive(Clone, Copy, Debug)] @@ -65,6 +66,7 @@ pub struct AudioSegmentTrack { get_stereo_mode: fn(&AudioConfiguration) -> StereoMode, get_offset: fn(&ClipOffsets) -> f32, timing_offset_secs: f32, + is_microphone: bool, } impl AudioSegmentTrack { @@ -94,9 +96,15 @@ impl AudioSegmentTrack { get_stereo_mode, get_offset, timing_offset_secs: 0.0, + is_microphone: false, } } + pub fn with_microphone_enhancement(mut self) -> Self { + self.is_microphone = true; + self + } + pub fn with_timing_offset_secs(mut self, timing_offset_secs: f32) -> Self { self.timing_offset_secs = timing_offset_secs; self @@ -135,6 +143,7 @@ struct SpeedAudioProcessorKey { segment_start_samples: usize, segment_end_samples: usize, mic_volume_bits: u32, + improve_microphone: bool, system_volume_bits: u32, mic_stereo_mode: u8, mic_offset_bits: u32, @@ -163,6 +172,7 @@ struct SpeedAudioProcessor { expected_source_sample: f64, timescale: f64, flushed: bool, + voice_enhancement: VoiceEnhancementCache, } impl AudioRenderer { @@ -188,6 +198,7 @@ impl AudioRenderer { transition_incoming: Vec::new(), speed_audio_processors: [None, None], speed_audio_use_counter: 0, + voice_enhancement: VoiceEnhancementCache::default(), } } @@ -199,6 +210,7 @@ impl AudioRenderer { pub fn set_playhead(&mut self, playhead: f64, project: &ProjectConfiguration) { self.elapsed_samples = self.playhead_to_samples(playhead); self.speed_audio_processors = [None, None]; + self.voice_enhancement = VoiceEnhancementCache::default(); self.cursor = match project.get_segment_time(playhead) { Some((segment_time, segment)) => AudioRendererCursor { @@ -654,7 +666,7 @@ impl AudioRenderer { } fn render_current_chunk( - &self, + &mut self, project: &ProjectConfiguration, samples: usize, out_offset: usize, @@ -720,6 +732,7 @@ impl AudioRenderer { segment_start_samples: self.playhead_to_samples(source.segment.start), segment_end_samples: self.playhead_to_samples(source.segment.end), mic_volume_bits: project.audio.mic_volume_db.to_bits(), + improve_microphone: project.audio.improve, system_volume_bits: project.audio.system_volume_db.to_bits(), mic_stereo_mode: project_stereo_mode_key(&project.audio.mic_stereo_mode), mic_offset_bits: offsets.mic.to_bits(), @@ -792,14 +805,22 @@ impl AudioRenderer { } fn render_chunk_at_cursor( - &self, + &mut self, project: &ProjectConfiguration, cursor: AudioRendererCursor, samples: usize, out_offset: usize, out: &mut [f32], ) -> usize { - render_audio_data_chunk(&self.data, project, cursor, samples, out_offset, out) + render_audio_data_chunk( + &self.data, + project, + cursor, + samples, + out_offset, + out, + &mut self.voice_enhancement, + ) } } @@ -818,6 +839,7 @@ fn render_audio_data_chunk( samples: usize, out_offset: usize, out: &mut [f32], + enhancement: &mut VoiceEnhancementCache, ) -> usize { let Some(segment) = data.get(cursor.clip_index as usize) else { return 0; @@ -869,7 +891,99 @@ fn render_audio_data_chunk( }) .collect::>(); - cap_audio::render_audio(&track_datas, cursor.samples, samples, out_offset, out) + if !project.audio.improve || project.audio.mute { + return cap_audio::render_audio(&track_datas, cursor.samples, samples, out_offset, out); + } + let clip_key = cursor.clip_index as usize; + for offset in (0..samples).step_by(EXPORT_AUDIO_BLOCK_SAMPLES) { + let count = (samples - offset).min(EXPORT_AUDIO_BLOCK_SAMPLES); + let cursor = cursor.samples + offset; + let enhanced = tracks + .iter() + .zip(&track_datas) + .enumerate() + .map(|(index, (track, data))| { + if !track.is_microphone + || !data.gain.is_finite() + || data.gain <= -30.0 + || !(1..=2).contains(&data.data.channels()) + { + return None; + } + let start = cursor as i128 + data.offset as i128; + let end = (start + count as i128).max(0) as usize; + let start = start.max(0) as usize; + if start >= end || start >= data.data.sample_count() { + return None; + } + Some(enhancement.render( + (clip_key, index), + data.data, + start, + end.saturating_sub(start), + )) + }) + .collect::>(); + let sources = track_datas + .iter() + .zip(&enhanced) + .map(|(track, enhanced)| { + enhanced + .as_ref() + .map_or(VoiceSource::Original(track.data), VoiceSource::Enhanced) + }) + .collect::>(); + let mixed = track_datas + .iter() + .zip(&sources) + .map(|(track, source)| AudioRendererTrack { + data: source, + gain: track.gain, + stereo_mode: match track.stereo_mode { + StereoMode::Stereo => StereoMode::Stereo, + StereoMode::MonoL => StereoMode::MonoL, + StereoMode::MonoR => StereoMode::MonoR, + }, + offset: track.offset, + }) + .collect::>(); + cap_audio::render_audio(&mixed, cursor, count, out_offset + offset * 2, out); + } + samples +} + +#[derive(Default)] +struct VoiceEnhancementCache { + slots: Vec<((usize, usize), VoiceEnhancer)>, +} + +impl VoiceEnhancementCache { + fn render( + &mut self, + key: (usize, usize), + source: &DecodedAudio, + start: usize, + count: usize, + ) -> VoiceAudio { + let index = self + .slots + .iter() + .position(|(candidate, state)| *candidate == key && state.is_contiguous(start)); + let slot = if let Some(index) = index { + self.slots.remove(index) + } else { + (key, VoiceEnhancer::new(source.channels())) + }; + if self.slots.len() == 4 { + self.slots.remove(0); + } + self.slots.push(slot); + self.slots + .last_mut() + .unwrap() + .1 + .render(source, start, count) + } } const SPEED_AUDIO_INPUT_BLOCK_SAMPLES: usize = 4_096; @@ -951,6 +1065,7 @@ impl SpeedAudioProcessor { expected_source_sample: requested_source_sample, timescale, flushed: false, + voice_enhancement: VoiceEnhancementCache::default(), }) } @@ -1017,6 +1132,7 @@ impl SpeedAudioProcessor { samples, 0, &mut self.input_data, + &mut self.voice_enhancement, ); let mut frame = FFAudio::new(AudioRenderer::SAMPLE_FORMAT, samples, ChannelLayout::STEREO); @@ -1509,6 +1625,24 @@ impl> PrerenderedAudioBuffer { Self { mode } } + pub(crate) fn bounded( + segments: Vec, + music: MusicTracks, + project: &ProjectConfiguration, + output_info: AudioInfo, + start_playhead_secs: f64, + ) -> Self { + Self { + mode: PrerenderedAudioBufferMode::Streaming(Box::new(StreamingAudioBuffer::new( + segments, + music, + project.clone(), + output_info, + start_playhead_secs, + ))), + } + } + pub fn wait_until_ready(&self, timeout: std::time::Duration) { match &self.mode { PrerenderedAudioBufferMode::Progressive(buffer) => buffer.wait_until_ready(timeout), @@ -2728,6 +2862,37 @@ mod tests { assert_eq!(renderer.speed_audio_processors.iter().flatten().count(), 2); } + #[test] + fn studio_sound_changes_only_the_microphone_and_preserves_system_audio() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("audio.wav"); + write_step_wav(&path, &[12_000]); + let data = Arc::new(DecodedAudio::from(Arc::new( + AudioData::from_file(&path).unwrap(), + ))); + for microphone in [false, true] { + let segment = crate::segments::audio_segment_from_decoded( + microphone.then(|| data.clone()), + (!microphone).then(|| data.clone()), + crate::SegmentAudioTimingRepair::default(), + ); + let mut project = ProjectConfiguration::default(); + let original = AudioRenderer::new(vec![segment.clone()]) + .render_frame_raw(48_000, &project) + .unwrap(); + project.audio.improve = true; + let enhanced = AudioRenderer::new(vec![segment]) + .render_frame_raw(48_000, &project) + .unwrap(); + assert_eq!(original.0, enhanced.0); + if microphone { + assert_ne!(original.1, enhanced.1); + } else { + assert_eq!(original.1, enhanced.1); + } + } + } + #[test] fn export_audio_virtual_negative_timing_offset_inserts_leading_silence() { let dir = TempDir::new().unwrap(); diff --git a/crates/editor/src/audio_output.rs b/crates/editor/src/audio_output.rs index e6863e5344..dfa6b5885a 100644 --- a/crates/editor/src/audio_output.rs +++ b/crates/editor/src/audio_output.rs @@ -56,13 +56,18 @@ pub struct PlaySpec { } enum ControlMsg { + Refresh { + spec: Box, + generation: u64, + retire_tx: std_mpsc::Sender, + }, + Retire(Box), EnsureStream, Play { spec: Box, generation: u64, result_tx: std_mpsc::Sender, }, - #[cfg(test)] PreparePlayback { spec: Box, generation: u64, @@ -188,6 +193,10 @@ pub(crate) struct PreparingAudioPlayTicket { } impl PreparingAudioPlayTicket { + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn output_handle(&self) -> Option { self.request .output @@ -283,6 +292,7 @@ impl Drop for PreparingAudioPlayTicket { } enum SourceAcknowledgement { + Refresh(std_mpsc::Sender), Ordinary(std_mpsc::Sender<()>), Preparing(Arc), } @@ -290,13 +300,14 @@ enum SourceAcknowledgement { impl SourceAcknowledgement { fn preparing_request(&self) -> Option> { match self { - Self::Ordinary(_) => None, + Self::Ordinary(_) | Self::Refresh(_) => None, Self::Preparing(request) => Some(request.clone()), } } fn consumed(self) { match &self { + Self::Refresh(_) => {} Self::Ordinary(sender) => { let _ = sender.send(()); } @@ -411,7 +422,6 @@ impl AudioOutput { } } - #[cfg(test)] pub(crate) fn prepare_playback(&self, spec: PlaySpec) -> PreparingAudioPlayTicket { let generation = self.next_generation.fetch_add(1, Ordering::Relaxed); let request = Arc::new(PreparingAudioRequest::new(PLAY_REQUEST_TIMEOUT)); @@ -464,6 +474,14 @@ impl AudioOutput { ticket } + pub(crate) fn refresh_playback(&self, spec: PlaySpec, generation: u64) { + let _ = self.control_tx.send(ControlMsg::Refresh { + spec: Box::new(spec), + generation, + retire_tx: self.control_tx.clone(), + }); + } + /// Detaches the source installed by the `play` call that returned this /// generation. A newer source (from a replacing play) is left untouched, /// so a stale playback shutting down can't cut off its successor. @@ -522,8 +540,14 @@ type InstallProgressiveAudio = dyn Fn(PreparingAudioSources, f64, u64, &PreparingAudioInstallation) -> Result<(), String>; enum SourceCommand { + Refresh { + source: Box>, + retire_tx: std_mpsc::Sender, + }, Install(Box>), - Remove { generation: Option }, + Remove { + generation: Option, + }, } /// The type-erased face of a running stream. The closures capture the typed @@ -548,6 +572,21 @@ fn control_thread(control_rx: std_mpsc::Receiver) { while let Ok(msg) = control_rx.recv() { match msg { + ControlMsg::Refresh { + spec, + generation, + retire_tx, + } => { + if let Some(stream) = &state + && let Err(error) = (stream.handle.install)( + spec, + generation, + SourceAcknowledgement::Refresh(retire_tx), + ) + { + error!(%error, "Could not update microphone enhancement during playback"); + } + } ControlMsg::EnsureStream => { ensure_stream(&mut state); } @@ -559,7 +598,6 @@ fn control_thread(control_rx: std_mpsc::Receiver) { let ok = handle_play(&mut state, spec, generation); let _ = result_tx.send(ok); } - #[cfg(test)] ControlMsg::PreparePlayback { spec, generation, @@ -615,6 +653,7 @@ fn control_thread(control_rx: std_mpsc::Receiver) { |stream, generation| (stream.handle.remove)(Some(generation)), ); } + ControlMsg::Retire(source) => drop(source), ControlMsg::Shutdown => break, } } @@ -643,6 +682,19 @@ fn drain_source_commands( ) { while let Ok(command) = source_rx.try_recv() { match command { + SourceCommand::Refresh { source, retire_tx } => { + let retired = if active + .as_ref() + .is_some_and(|current| current.generation == source.generation) + { + active.replace(*source).map(Box::new) + } else { + Some(source) + }; + if let Some(retired) = retired { + let _ = retire_tx.send(ControlMsg::Retire(retired)); + } + } SourceCommand::Install(source) => { if !source .preparing_request @@ -796,15 +848,21 @@ fn install_source>( }; let start_playhead = start_playhead_secs + initial_latency_secs; - let mut buffer = PrerenderedAudioBuffer::::new( - segments, - music, - &project, - output_info, - duration_secs, - start_playhead, - ); - buffer.set_playhead(start_playhead); + let mut buffer = if matches!(ack, SourceAcknowledgement::Refresh(_)) { + PrerenderedAudioBuffer::::bounded(segments, music, &project, output_info, start_playhead) + } else { + PrerenderedAudioBuffer::::new( + segments, + music, + &project, + output_info, + duration_secs, + start_playhead, + ) + }; + if !matches!(ack, SourceAcknowledgement::Refresh(_)) { + buffer.set_playhead(start_playhead); + } // A few ms: guarantees the callback reads real samples at the // playhead, never leading silence. buffer.wait_until_ready(PRERENDER_READY_TIMEOUT); @@ -820,16 +878,25 @@ fn install_source>( request.awaiting_callback(); } + let retire_tx = match &ack { + SourceAcknowledgement::Refresh(sender) => Some(sender.clone()), + _ => None, + }; + let source = Box::new(ActiveSource { + generation, + buffer: ActiveSourceBuffer::Ordinary(buffer), + playhead_rx, + ack: Some(ack), + preparing_request, + #[cfg(not(target_os = "windows"))] + latency_corrector, + }); install_tx - .send(SourceCommand::Install(Box::new(ActiveSource { - generation, - buffer: ActiveSourceBuffer::Ordinary(buffer), - playhead_rx, - ack: Some(ack), - preparing_request, - #[cfg(not(target_os = "windows"))] - latency_corrector, - }))) + .send(if let Some(retire_tx) = retire_tx { + SourceCommand::Refresh { source, retire_tx } + } else { + SourceCommand::Install(source) + }) .map_err(|_| "Audio callback channel closed".to_string()) } @@ -941,6 +1008,22 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: while let Ok(msg) = control_rx.recv() { match msg { + ControlMsg::Refresh { + spec, + generation, + retire_tx, + } => { + if let Err(error) = install_source::( + spec, + generation, + SourceAcknowledgement::Refresh(retire_tx), + output_info, + false, + &source_tx, + ) { + error!(%error, "Could not update headless audio playback"); + } + } ControlMsg::EnsureStream => {} ControlMsg::Play { spec, @@ -965,7 +1048,6 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: }; let _ = result_tx.send(ok); } - #[cfg(test)] ControlMsg::PreparePlayback { spec, generation, @@ -1016,6 +1098,7 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: generation: Some(generation), }); } + ControlMsg::Retire(source) => drop(source), ControlMsg::Shutdown => break, } } @@ -1273,6 +1356,53 @@ mod tests { } } + #[test] + fn enhancement_refresh_only_replaces_its_active_playback_generation() { + let (mut original, _playhead) = source(48_000); + original.generation = 7; + let mut active = Some(original); + let (tx, rx) = std_mpsc::channel(); + let (retire_tx, retire_rx) = std_mpsc::channel(); + for (generation, should_replace) in [(7, true), (6, false), (8, false)] { + let (mut replacement, _playhead) = source(48_000); + replacement.generation = generation; + let ActiveSourceBuffer::Ordinary(buffer) = &mut replacement.buffer else { + panic!("expected ordinary source"); + }; + buffer.set_playhead(0.5); + let ActiveSourceBuffer::Ordinary(buffer) = &mut active.as_mut().unwrap().buffer else { + panic!("expected ordinary source"); + }; + buffer.set_playhead(0.0); + tx.send(SourceCommand::Refresh { + source: Box::new(replacement), + retire_tx: retire_tx.clone(), + }) + .unwrap(); + drain_source_commands(&mut active, &rx); + assert!(matches!( + retire_rx.try_recv().unwrap(), + ControlMsg::Retire(_) + )); + let actual = active.as_ref().unwrap(); + assert_eq!(actual.generation, 7); + assert_eq!(actual.buffer.current_playhead_secs() >= 0.5, should_replace); + } + tx.send(SourceCommand::Remove { + generation: Some(7), + }) + .unwrap(); + let (mut replacement, _playhead) = source(48_000); + replacement.generation = 7; + tx.send(SourceCommand::Refresh { + source: Box::new(replacement), + retire_tx: retire_tx.clone(), + }) + .unwrap(); + drain_source_commands(&mut active, &rx); + assert!(active.is_none()); + } + #[test] fn coalesced_video_updates_preserve_synchronized_audio_position() { for sample_rate in [44_100, 48_000] { diff --git a/crates/editor/src/audio_output/native_tests.rs b/crates/editor/src/audio_output/native_tests.rs index a9e81e7cd2..0798a23c35 100644 --- a/crates/editor/src/audio_output/native_tests.rs +++ b/crates/editor/src/audio_output/native_tests.rs @@ -712,3 +712,36 @@ async fn progressive_output_without_a_prefix_never_acknowledges_and_cancels_clea .unwrap() .unwrap(); } + +#[test] +fn studio_sound_refresh_changes_live_pcm_and_cannot_restart_stopped_audio() { + let (tx, rx) = std_mpsc::channel(); + let output = AudioOutput::new_headless(Box::new(move |samples, _| { + let mean = samples.iter().map(|sample| sample.abs()).sum::() / samples.len() as f32; + let _ = tx.send(mean); + })); + let generation = output.play(audible_spec()).unwrap(); + let wait_for = |predicate: fn(f32) -> bool| { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let value = rx + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .unwrap(); + if predicate(value) { + break; + } + } + }; + wait_for(|mean| mean > 0.1); + let mut enhanced = audible_spec(); + enhanced.project.audio.improve = true; + output.refresh_playback(enhanced, generation); + wait_for(|mean| mean > 0.005 && mean < 0.04); + output.stop_playback(generation); + wait_for(|mean| mean == 0.0); + output.refresh_playback(audible_spec(), generation); + for _ in 0..30 { + assert_eq!(rx.recv_timeout(Duration::from_secs(2)).unwrap(), 0.0); + } + output.shutdown(); +} diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index f4568f7b1e..a668af23aa 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -888,6 +888,16 @@ impl EditorInstance { tokio::spawn(async move { loop { let event = *handle.receive_event().await; + if this.playback_epoch.load(Ordering::SeqCst) == epoch + && handle.preparing_audio_released() + { + drop( + this.preparing_adoption + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(), + ); + } match event { playback::PlaybackEvent::Start => {} diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rs index 1f18fad041..a01c07c86e 100644 --- a/crates/editor/src/export_audio.rs +++ b/crates/editor/src/export_audio.rs @@ -1,6 +1,7 @@ use crate::{AudioRenderer, SegmentMedia}; use cap_audio::{ AudioRendererTrack, AudioSampleSource, AudioStream, AudioStreamError, ChunkRead, StereoMode, + VOICE_PREROLL_SAMPLES, VOICE_WINDOW_PADDING_SAMPLES, VoiceAudio, VoiceEnhancer, VoiceSource, }; use cap_project::{ClipOffsets, ProjectConfiguration, RecordingMeta, StudioRecordingMeta}; use std::{ @@ -15,6 +16,8 @@ use std::{ pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096; const MAX_PARALLEL_AUDIO_SOURCES: usize = 4; +const MAX_ENHANCED_WINDOW_SAMPLES: usize = + EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES; #[derive(Clone, Debug)] pub enum ExportAudioError { @@ -466,9 +469,43 @@ impl ExportAudioSources { .unwrap_or_default(); for track in tracks.iter_mut() { let start = cursor as i128 + track.offset(&offsets) as i128; - track.prepare(start, start + samples as i128)?; + let end = start + samples as i128; + let improve = track.mic + && project.audio.improve + && !project.audio.mute + && project.audio.mic_volume_db > -30.0 + && (1..=2).contains(&track.source.channels()); + if improve { + let start = + usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; + let end = + usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; + let mut enhancer = track + .enhancer + .take() + .unwrap_or_else(|| VoiceEnhancer::new(track.source.channels())); + let range = enhancer.source_range(start, end.saturating_sub(start)); + track.prepare(range.start as i128, range.end as i128)?; + track.enhanced = + Some(enhancer.render(&track.view(), start, end.saturating_sub(start))); + track.enhancer = Some(enhancer); + } else { + track.prepare(start, end)?; + track.enhanced = None; + track.enhancer = None; + } } let views = tracks.iter().map(|track| track.view()).collect::>(); + let sources = tracks + .iter() + .zip(&views) + .map(|(track, view)| { + track + .enhanced + .as_ref() + .map_or(VoiceSource::Original(view), VoiceSource::Enhanced) + }) + .collect::>(); let max_samples = tracks .iter() .map(|track| (track.available_end as isize - track.offset(&offsets)).max(0) as usize) @@ -479,7 +516,7 @@ impl ExportAudioSources { } let tracks = tracks .iter() - .zip(&views) + .zip(&sources) .map(|(track, view)| { let gain = if track.mic { project.audio.mic_volume_db @@ -526,6 +563,8 @@ struct ExportAudioTrack { source_start: usize, available_end: usize, eof: Option, + enhancer: Option, + enhanced: Option, } impl ExportAudioTrack { @@ -553,6 +592,8 @@ impl ExportAudioTrack { source_start: 0, available_end: 0, eof: None, + enhancer: None, + enhanced: None, }) } @@ -568,7 +609,7 @@ impl ExportAudioTrack { fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> { let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; - if end < start || end - start > EXPORT_AUDIO_BLOCK_SAMPLES || start < self.source_start { + if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start { return Err(ExportAudioError::InvalidWindow); } let channels = self.source.channels() as usize; @@ -1063,7 +1104,8 @@ mod tests { cap_project::StereoMode::MonoR => StereoMode::MonoR, }, |offset| offset.mic, - ), + ) + .with_microphone_enhancement(), AudioSegmentTrack::new( data[1].clone(), |config| config.system_volume_db, @@ -1073,16 +1115,19 @@ mod tests { ], }, AudioSegment { - tracks: vec![AudioSegmentTrack::new( - data[2].clone(), - |config| config.mic_volume_db, - |config| match config.mic_stereo_mode { - cap_project::StereoMode::Stereo => StereoMode::Stereo, - cap_project::StereoMode::MonoL => StereoMode::MonoL, - cap_project::StereoMode::MonoR => StereoMode::MonoR, - }, - |offset| offset.mic, - )], + tracks: vec![ + AudioSegmentTrack::new( + data[2].clone(), + |config| config.mic_volume_db, + |config| match config.mic_stereo_mode { + cap_project::StereoMode::Stereo => StereoMode::Stereo, + cap_project::StereoMode::MonoL => StereoMode::MonoL, + cap_project::StereoMode::MonoR => StereoMode::MonoR, + }, + |offset| offset.mic, + ) + .with_microphone_enhancement(), + ], }, ]; let tracks = vec![ @@ -1272,6 +1317,78 @@ mod tests { assert_eq!(candidate.sources.tracks.iter().flatten().count(), 5); } + #[test] + #[ignore = "requires CAP_STUDIO_SOUND_BENCH_RECORDING pointing to a local Studio recording"] + fn studio_sound_streams_a_real_recording_with_bounded_memory() { + ffmpeg::init().unwrap(); + let path = std::env::var_os("CAP_STUDIO_SOUND_BENCH_RECORDING").unwrap(); + let recording = RecordingMeta::load_for_project(Path::new(&path)).unwrap(); + let mut totals = Vec::new(); + for improve in [false, true] { + let started = std::time::Instant::now(); + let mut preparation = ExportAudioPreparation::open( + &recording, + recording.studio_meta().unwrap(), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + let mut project = ProjectConfiguration::default(); + project.audio.improve = improve; + let mut count = 0_usize; + let mut peak_window_bytes = 0; + let mut output = [0.0; EXPORT_AUDIO_BLOCK_SAMPLES * 2]; + for clip in 0..preparation.sources.tracks.len() { + let mut cursor = 0; + loop { + output.fill(0.0); + let rendered = preparation + .sources + .render( + &project, + clip as u32, + cursor, + EXPORT_AUDIO_BLOCK_SAMPLES, + &mut output, + ) + .unwrap(); + assert!( + output + .iter() + .all(|sample| sample.is_finite() && sample.abs() <= 1.0) + ); + let bytes = preparation + .sources + .tracks + .iter() + .flatten() + .map(|track| { + assert!( + track.samples.len() / usize::from(track.source.channels()) + <= MAX_ENHANCED_WINDOW_SAMPLES + ); + track.samples.capacity() * size_of::() + }) + .sum::(); + peak_window_bytes = peak_window_bytes.max(bytes); + cursor += rendered; + if rendered == 0 { + break; + } + } + count += cursor; + } + assert!(count > 0); + totals.push(count); + eprintln!( + "studio_sound={improve}, audio_seconds={:.3}, elapsed_seconds={:.3}, source_window_bytes={peak_window_bytes}", + count as f64 / 48_000.0, + started.elapsed().as_secs_f64() + ); + } + assert_eq!(totals[0], totals[1]); + } + #[test] fn bounded_sink_matches_full_renderer_at_original_request_boundaries() { ffmpeg::init().unwrap(); @@ -1285,7 +1402,7 @@ mod tests { .iter() .map(|path| Arc::new(AudioData::from_file(path).unwrap())) .collect::>(); - for variant in 0..16 { + for variant in 0..20 { let mut project = project(); match variant { 0 => project.timeline = None, @@ -1327,6 +1444,28 @@ mod tests { project.audio.mic_volume_db = 4.0; project.audio.system_volume_db = -29.9; } + 16 => project.audio.improve = true, + 17 => { + project.audio.improve = true; + project.timeline = None; + } + 18 => { + project.audio.improve = true; + project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR; + } + 19 => { + project.audio.improve = true; + project + .timeline + .as_mut() + .unwrap() + .transitions + .push(ClipTransition { + segment_index: 1, + kind: ClipTransitionType::CrossFade, + duration: 0.131_234_567, + }); + } _ => unreachable!(), } for request in [1, 7, 997, 4_096, 4_800, 48_001, 96_000, 384_000] { @@ -1371,7 +1510,11 @@ mod tests { for track in candidate.sources.tracks.iter().flatten() { assert!( track.samples.len() - <= EXPORT_AUDIO_BLOCK_SAMPLES * track.source.channels() as usize + <= (if track.enhancer.is_some() { + MAX_ENHANCED_WINDOW_SAMPLES + } else { + EXPORT_AUDIO_BLOCK_SAMPLES + }) * track.source.channels() as usize ); } if expected.is_none() { diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 1dc66ab731..86faa3003a 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -14,7 +14,7 @@ use std::{ num::NonZeroUsize, sync::{ Arc, RwLock, - atomic::{AtomicU64, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, mpsc as std_mpsc, }, time::{Duration, Instant}, @@ -134,6 +134,7 @@ pub enum PlaybackEvent { #[derive(Clone)] pub struct PlaybackHandle { + preparing_audio_released: Arc, stop_tx: watch::Sender, event_rx: watch::Receiver, seek_tx: Arc>, @@ -579,7 +580,10 @@ impl Playback { let (seek_tx, mut seek_rx) = watch::channel((0u64, self.start_frame_number)); seek_rx.borrow_and_update(); + let preparing_audio_released = Arc::new(AtomicBool::new(false)); + let runtime = tokio::runtime::Handle::current(); let handle = PlaybackHandle { + preparing_audio_released: preparing_audio_released.clone(), stop_tx: stop_tx.clone(), event_rx, seek_tx: Arc::new(seek_tx), @@ -860,8 +864,8 @@ impl Playback { )); diagnostic.stage("preparing_playback"); let playback_body = move || { - let adopted_guard = AdoptedPlaybackGuard(adopted); - let adopted = &adopted_guard.0; + let mut adopted_guard = AdoptedPlaybackGuard(adopted); + let adopted = &mut adopted_guard.0; let duration = self .project .borrow() @@ -1248,22 +1252,22 @@ impl Playback { diagnostic.stage("initializing_audio_output"); let audio_spawn_start = Instant::now(); let _ = audio_playhead_tx.send(playback_start_frame as f64 / fps_f64); - let audio_generation = if adopted.is_some() { + let mut audio_generation = if adopted.is_some() { None } else if !has_playback_audio(&audio_segments, !self.music.is_empty()) { info!("No audio segments found, skipping audio playback."); None } else { self.audio_output.play(PlaySpec { - segments: audio_segments, + segments: audio_segments.clone(), music: self.music.clone(), project: self.project.borrow().clone(), duration_secs: duration, start_playhead_secs: playback_start_frame as f64 / fps_f64, - playhead_rx: audio_playhead_rx, + playhead_rx: audio_playhead_rx.clone(), }) }; - let has_audio = audio_generation.is_some(); + let mut has_audio = audio_generation.is_some(); if let Some(telemetry) = &self.telemetry { telemetry.emit(PlaybackTelemetryEvent::AudioPipelineReady { elapsed: audio_spawn_start.elapsed(), @@ -1286,11 +1290,57 @@ impl Playback { let mut start = Instant::now(); let mut clock_anchor_frame = playback_start_frame; + let mut transitioned_audio = None; let mut last_adopted_frame = None; 'playback: loop { if *stop_rx.borrow() { break; } + if self.project.borrow().audio.improve + && adopted.as_ref().is_some_and(|adoption| adoption.is_owner()) + { + let adoption = adopted.as_ref().unwrap(); + frame_number = adoption.frame_number(fps).unwrap_or(frame_number); + let cleaned = runtime.block_on(async { + tokio::select! { + biased; + _ = stop_rx.changed() => false, + result = tokio::time::timeout(Duration::from_secs(2), adoption.stop_and_wait()) => { + matches!(result, Ok(Some(exit)) if !exit.cleanup_failed) + } + } + }); + if !cleaned || *stop_rx.borrow() { + break; + } + drop(adopted.take()); + preparing_audio_released.store(true, Ordering::Release); + let project = self.project.borrow().clone(); + cached_project.audio.improve = project.audio.improve; + let ticket = self.audio_output.prepare_playback(PlaySpec { + segments: audio_segments.clone(), + music: self.music.clone(), + project, + duration_secs: duration, + start_playhead_secs: frame_number as f64 / fps_f64, + playhead_rx: audio_playhead_rx.clone(), + }); + let started = runtime.block_on(async { + tokio::select! { + biased; + _ = stop_rx.changed() => false, + result = tokio::time::timeout(Duration::from_secs(2), ticket.wait_started()) => result.unwrap_or(false), + } + }); + if !started || *stop_rx.borrow() { + break; + } + audio_generation = Some(ticket.generation()); + transitioned_audio = Some(ticket); + has_audio = true; + clock_anchor_frame = frame_number; + start = Instant::now(); + } if let Some(adoption) = adopted { let Some(snapshot) = adoption.snapshot() else { break; @@ -1334,7 +1384,24 @@ impl Playback { } if self.project.has_changed().unwrap_or(false) { + let improved = cached_project.audio.improve; cached_project = self.project.borrow_and_update().clone(); + if adopted.is_none() + && improved != cached_project.audio.improve + && let Some(generation) = audio_generation + { + self.audio_output.refresh_playback( + PlaySpec { + segments: audio_segments.clone(), + music: self.music.clone(), + project: cached_project.clone(), + duration_secs: duration, + start_playhead_secs: frame_number as f64 / fps_f64, + playhead_rx: audio_playhead_rx.clone(), + }, + generation, + ); + } cursor_timelines = build_cursor_timelines(&cached_project); zoom_timelines = build_zoom_timelines(&cached_project); outgoing_zoom_timelines = build_outgoing_zoom_timelines(&cached_project); @@ -1884,6 +1951,7 @@ impl Playback { self.audio_output.stop_playback(generation); } + drop(transitioned_audio); stop_tx.send(true).ok(); event_tx.send(PlaybackEvent::Stop).ok(); @@ -1909,6 +1977,10 @@ impl Drop for AdoptedPlaybackGuard { } impl PlaybackHandle { + pub(crate) fn preparing_audio_released(&self) -> bool { + self.preparing_audio_released.load(Ordering::Acquire) + } + pub fn stop(&self) { if let Some(adoption) = &self.adopted { adoption.cancel(); @@ -1920,7 +1992,7 @@ impl PlaybackHandle { /// re-attach. Returns false once the playback thread is gone, which is the /// caller's cue to fall back to a full restart. pub fn seek(&self, frame: u32) -> bool { - if self.adopted.is_some() { + if self.adopted.is_some() && !self.preparing_audio_released() { return false; } let generation = self.seek_generation.fetch_add(1, Ordering::Relaxed) + 1; @@ -1945,6 +2017,7 @@ mod tests { let (seek_tx, _) = watch::channel((0, 0)); ( PlaybackHandle { + preparing_audio_released: Arc::new(AtomicBool::new(false)), adopted: None, stop_tx, event_rx, diff --git a/crates/editor/src/preparing_preview/native_tests.rs b/crates/editor/src/preparing_preview/native_tests.rs index 81044c0585..34b661f487 100644 --- a/crates/editor/src/preparing_preview/native_tests.rs +++ b/crates/editor/src/preparing_preview/native_tests.rs @@ -1421,3 +1421,148 @@ async fn native_cancelled_handoff_installation_joins_and_releases_the_candidate( async fn native_rejected_audio_handoff_installation_joins_and_releases_the_candidate() { failed_handoff_installation_retires_native_candidate(false).await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_studio_sound_releases_preparing_audio_and_preserves_transport() { + for initially_enabled in [false, true] { + let mut fixture = handoff_fixture(); + let cap_project::RecordingMetaInner::Studio(studio) = &mut fixture.metadata.inner else { + panic!("Expected Studio metadata"); + }; + let cap_project::StudioRecordingMeta::MultipleSegments { inner } = studio.as_mut() else { + panic!("Expected indexed metadata"); + }; + for segment in &mut inner.segments { + segment.display.path = "display".into(); + segment.camera.as_mut().unwrap().path = "camera".into(); + } + fixture.metadata.save_for_project().unwrap(); + let output = Arc::new(crate::AudioOutput::new_headless(Box::new(|_, _| {}))); + let session = crate::PreparingPlaybackSession::spawn( + fixture.input(), + (0..3) + .map(|_| crate::PreparingAudioSegmentInput { + mic: Some( + cap_audio::ManagedAudioInput::new( + fixture.source.clone(), + PathBuf::from("mic.aac"), + ) + .unwrap(), + ), + system_audio: None, + timing_repair: Default::default(), + }) + .collect(), + crate::PreparingPlaybackOptions { + preview: PreparingPreviewOptions::default(), + fps: 30, + resolution: XY::new(320, 240), + }, + output.clone(), + Box::new(|_, _, _| {}), + ) + .unwrap(); + let handoff = session.handoff_handle(); + let cache = tokio::time::timeout(Duration::from_secs(30), handoff.take_completed_audio()) + .await + .unwrap() + .unwrap(); + let (frames, mut received) = tokio::sync::watch::channel(None); + let candidate = crate::EditorInstance::new_with_startup_inputs( + fixture.metadata.project_path.clone(), + |_| {}, + Box::new(move |output, _| { + let frame = match output { + crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, + #[cfg(target_os = "macos")] + crate::EditorFrameOutput::Surface(frame) => frame.frame_number, + }; + frames.send_replace(Some(frame)); + }), + None, + crate::EditorFrameFormat::Rgba, + output, + crate::EditorStartupInputs { + recordings: None, + completed_audio: Some(cache), + }, + ) + .await + .unwrap(); + candidate + .project_config + .0 + .send_modify(|project| project.audio.improve = initially_enabled); + let mut updates = session.updates(); + tokio::time::timeout(Duration::from_secs(30), async { + while !updates.borrow_and_update().progress.preview_available { + updates.changed().await.unwrap(); + } + }) + .await + .unwrap(); + candidate + .preview_tx + .send(Some((0, 30, XY::new(320, 240)))) + .unwrap(); + tokio::time::timeout(Duration::from_secs(30), async { + while received.borrow_and_update().is_none() { + received.changed().await.unwrap(); + } + }) + .await + .unwrap(); + session.controller().seek(0.5).await.unwrap(); + session.controller().set_playing(true).await.unwrap(); + candidate.install_preparing_handoff(&handoff).await.unwrap(); + assert!( + candidate + .start_preparing_handoff(30, XY::new(320, 240)) + .await + .unwrap() + ); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if received + .borrow_and_update() + .is_some_and(|frame| candidate.commit_preparing_frame(frame, 30)) + { + break; + } + received.changed().await.unwrap(); + } + }) + .await + .expect("Prepared editor did not present a handoff frame"); + let external_handle = candidate.state.lock().await.playback_task.clone().unwrap(); + candidate + .project_config + .0 + .send_modify(|project| project.audio.improve = true); + tokio::time::timeout(Duration::from_secs(10), async { + while candidate.preparing_adoption().is_some() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + while !external_handle.seek(18) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + while candidate.state.lock().await.playhead_position < 18 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let mut active = candidate.playback_watch(); + assert!(*active.borrow_and_update()); + external_handle.stop(); + tokio::time::timeout(Duration::from_secs(5), async { + while *active.borrow_and_update() { + active.changed().await.unwrap(); + } + }) + .await + .unwrap(); + candidate.dispose().await; + } +} diff --git a/crates/editor/src/segments.rs b/crates/editor/src/segments.rs index 7c8f8f371d..60953e588f 100644 --- a/crates/editor/src/segments.rs +++ b/crates/editor/src/segments.rs @@ -150,6 +150,7 @@ pub fn audio_segment_from_decoded( }, |o| o.mic, ) + .with_microphone_enhancement() .with_timing_offset_secs(repair.mic_offset_secs) }), system_audio.map(|a| -> AudioSegmentTrack { From 3b03c7d793077c3cf1cec05176270e46df8d7f0a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:56:51 +0100 Subject: [PATCH 2/2] fix: preserve audio position when toggling Studio Sound --- crates/editor/src/audio.rs | 85 +++++++++- crates/editor/src/audio_output.rs | 153 ++++++++++++++++-- .../editor/src/audio_output/native_tests.rs | 10 +- 3 files changed, 222 insertions(+), 26 deletions(-) diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 942727523c..ac363c19c4 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -1665,6 +1665,18 @@ impl> PrerenderedAudioBuffer { } } + pub(crate) fn try_align_prepared_playhead(&mut self, playhead_secs: f64) -> bool { + match &mut self.mode { + PrerenderedAudioBufferMode::Progressive(buffer) => { + buffer.set_playhead(playhead_secs); + true + } + PrerenderedAudioBufferMode::Streaming(buffer) => { + buffer.try_align_prepared_playhead(playhead_secs) + } + } + } + pub fn current_audible_playhead(&self, device_latency_secs: f64) -> f64 { match &self.mode { PrerenderedAudioBufferMode::Progressive(buffer) => { @@ -1860,6 +1872,7 @@ struct StreamingAudioBuffer { project: ProjectConfiguration, sample_rate: u32, channels: usize, + read_position: usize, } impl StreamingAudioBuffer { @@ -1887,6 +1900,7 @@ impl StreamingAudioBuffer { project, sample_rate: output_info.sample_rate, channels: output_info.channels, + read_position: 0, }; buffer.set_playhead(start_playhead_secs); buffer @@ -1904,20 +1918,38 @@ impl StreamingAudioBuffer { fn set_playhead(&mut self, playhead_secs: f64) { self.resampler.reset(); self.resampled_buffer.clear(); + self.read_position = + output_sample_index(playhead_secs, self.sample_rate, self.channels, usize::MAX); self.renderer.set_playhead(playhead_secs, &self.project); self.prefill(self.ready_window_samples()); } + fn try_align_prepared_playhead(&mut self, playhead_secs: f64) -> bool { + let delta = playhead_secs - self.current_audible_playhead(0.0); + let samples = ((delta.max(0.0) * f64::from(self.sample_rate)).round() as usize) + .saturating_mul(self.channels); + if delta.is_finite() + && delta >= -1.0 / f64::from(self.sample_rate) + && samples <= self.resampled_buffer.occupied_len() + { + self.read_position = self + .read_position + .saturating_add(self.resampled_buffer.skip(samples)); + true + } else { + false + } + } + fn current_audible_playhead(&self, device_latency_secs: f64) -> f64 { - let generated_secs = self.renderer.elapsed_samples_to_playhead(); - let buffered_frames = self.resampled_buffer.occupied_len() / self.channels; - let buffered_secs = buffered_frames as f64 / self.sample_rate as f64; - (generated_secs - buffered_secs - device_latency_secs.max(0.0)).max(0.0) + let consumed_secs = + (self.read_position / self.channels) as f64 / f64::from(self.sample_rate); + (consumed_secs - device_latency_secs.max(0.0)).max(0.0) } #[allow(dead_code)] fn current_playhead_secs(&self) -> f64 { - self.renderer.elapsed_samples_to_playhead() + self.current_audible_playhead(0.0) } fn buffer_reaching_limit(&self) -> bool { @@ -1972,6 +2004,7 @@ impl StreamingAudioBuffer { } let filled = self.resampled_buffer.pop_slice(playback_buffer); + self.read_position = self.read_position.saturating_add(filled); playback_buffer[filled..].fill(T::EQUILIBRIUM); self.prefill(self.ready_window_samples().max(playback_buffer.len())); @@ -2862,6 +2895,48 @@ mod tests { assert_eq!(renderer.speed_audio_processors.iter().flatten().count(), 2); } + #[test] + fn studio_sound_prepared_alignment_reuses_samples_without_rendering() { + let (_dir, mut renderer, mut project) = build_renderer_fixture(); + renderer.data[0].tracks[0].is_microphone = true; + project.audio.improve = true; + for sample_rate in [44_100, 48_000] { + let info = AudioInfo::new(AudioRenderer::SAMPLE_FORMAT, sample_rate, 2).unwrap(); + let make = || { + StreamingAudioBuffer::::new( + renderer.data.clone(), + MusicTracks::new(), + project.clone(), + info, + 0.0, + ) + }; + let mut reference = make(); + let mut candidate = make(); + reference.fill(&mut vec![0.0; sample_rate as usize / 8 * 2]); + let target = reference.current_audible_playhead(0.0); + let rendered_before = candidate.renderer.elapsed_samples; + assert!(candidate.try_align_prepared_playhead(target)); + assert_eq!(candidate.renderer.elapsed_samples, rendered_before); + assert!( + (candidate.current_audible_playhead(0.0) - target).abs() + <= 1.0 / f64::from(sample_rate) + ); + let mut expected = [0.0; 512 * 2]; + let mut actual = [0.0; 512 * 2]; + reference.fill(&mut expected); + candidate.fill(&mut actual); + assert_eq!(actual, expected); + for target in [1.0, 0.0] { + let before = candidate.renderer.elapsed_samples; + assert!(!candidate.try_align_prepared_playhead(target)); + assert_eq!(candidate.renderer.elapsed_samples, before); + candidate.set_playhead(target); + assert!((candidate.current_audible_playhead(0.0) - target).abs() < 0.01); + } + } + } + #[test] fn studio_sound_changes_only_the_microphone_and_preserves_system_audio() { let dir = TempDir::new().unwrap(); diff --git a/crates/editor/src/audio_output.rs b/crates/editor/src/audio_output.rs index dfa6b5885a..3f3ae6b611 100644 --- a/crates/editor/src/audio_output.rs +++ b/crates/editor/src/audio_output.rs @@ -59,9 +59,11 @@ enum ControlMsg { Refresh { spec: Box, generation: u64, + revision: u64, retire_tx: std_mpsc::Sender, }, Retire(Box), + Realign(Box), EnsureStream, Play { spec: Box, @@ -292,7 +294,7 @@ impl Drop for PreparingAudioPlayTicket { } enum SourceAcknowledgement { - Refresh(std_mpsc::Sender), + Refresh(std_mpsc::Sender, u64), Ordinary(std_mpsc::Sender<()>), Preparing(Arc), } @@ -300,14 +302,14 @@ enum SourceAcknowledgement { impl SourceAcknowledgement { fn preparing_request(&self) -> Option> { match self { - Self::Ordinary(_) | Self::Refresh(_) => None, + Self::Ordinary(_) | Self::Refresh(..) => None, Self::Preparing(request) => Some(request.clone()), } } fn consumed(self) { match &self { - Self::Refresh(_) => {} + Self::Refresh(..) => {} Self::Ordinary(sender) => { let _ = sender.send(()); } @@ -478,6 +480,7 @@ impl AudioOutput { let _ = self.control_tx.send(ControlMsg::Refresh { spec: Box::new(spec), generation, + revision: self.next_generation.fetch_add(1, Ordering::Relaxed), retire_tx: self.control_tx.clone(), }); } @@ -506,6 +509,7 @@ impl Drop for AudioOutput { /// Per-playback state owned by the audio callback. struct ActiveSource { generation: u64, + refresh_revision: u64, buffer: ActiveSourceBuffer, playhead_rx: watch::Receiver, ack: Option, @@ -543,6 +547,7 @@ enum SourceCommand { Refresh { source: Box>, retire_tx: std_mpsc::Sender, + requeue_tx: std_mpsc::Sender>, }, Install(Box>), Remove { @@ -575,13 +580,14 @@ fn control_thread(control_rx: std_mpsc::Receiver) { ControlMsg::Refresh { spec, generation, + revision, retire_tx, } => { if let Some(stream) = &state && let Err(error) = (stream.handle.install)( spec, generation, - SourceAcknowledgement::Refresh(retire_tx), + SourceAcknowledgement::Refresh(retire_tx, revision), ) { error!(%error, "Could not update microphone enhancement during playback"); @@ -654,6 +660,7 @@ fn control_thread(control_rx: std_mpsc::Receiver) { ); } ControlMsg::Retire(source) => drop(source), + ControlMsg::Realign(realign) => realign(), ControlMsg::Shutdown => break, } } @@ -676,17 +683,44 @@ fn stop_stream_state( /// Applies pending install/remove commands to the active source. Shared by /// the live cpal callback and the headless sink. -fn drain_source_commands( +fn drain_source_commands>( active: &mut Option>, source_rx: &std_mpsc::Receiver>, ) { while let Ok(command) = source_rx.try_recv() { match command { - SourceCommand::Refresh { source, retire_tx } => { - let retired = if active - .as_ref() - .is_some_and(|current| current.generation == source.generation) - { + SourceCommand::Refresh { + mut source, + retire_tx, + requeue_tx, + } => { + let retired = if let Some(current) = active.as_mut().filter(|current| { + current.generation == source.generation + && current.refresh_revision <= source.refresh_revision + }) { + current.refresh_revision = source.refresh_revision; + if let ( + ActiveSourceBuffer::Ordinary(previous), + ActiveSourceBuffer::Ordinary(replacement), + ) = (¤t.buffer, &mut source.buffer) + { + let playhead = previous.current_audible_playhead(0.0); + if !replacement.try_align_prepared_playhead(playhead) { + let sender = retire_tx.clone(); + let _ = sender.send(ControlMsg::Realign(Box::new(move || { + if let ActiveSourceBuffer::Ordinary(buffer) = &mut source.buffer { + buffer.set_playhead(playhead); + } + let sender = requeue_tx.clone(); + let _ = sender.send(SourceCommand::Refresh { + source, + retire_tx, + requeue_tx, + }); + }))); + continue; + } + } active.replace(*source).map(Box::new) } else { Some(source) @@ -848,7 +882,7 @@ fn install_source>( }; let start_playhead = start_playhead_secs + initial_latency_secs; - let mut buffer = if matches!(ack, SourceAcknowledgement::Refresh(_)) { + let mut buffer = if matches!(ack, SourceAcknowledgement::Refresh(..)) { PrerenderedAudioBuffer::::bounded(segments, music, &project, output_info, start_playhead) } else { PrerenderedAudioBuffer::::new( @@ -860,7 +894,7 @@ fn install_source>( start_playhead, ) }; - if !matches!(ack, SourceAcknowledgement::Refresh(_)) { + if !matches!(ack, SourceAcknowledgement::Refresh(..)) { buffer.set_playhead(start_playhead); } // A few ms: guarantees the callback reads real samples at the @@ -879,11 +913,16 @@ fn install_source>( } let retire_tx = match &ack { - SourceAcknowledgement::Refresh(sender) => Some(sender.clone()), + SourceAcknowledgement::Refresh(sender, _) => Some(sender.clone()), _ => None, }; + let refresh_revision = match &ack { + SourceAcknowledgement::Refresh(_, revision) => *revision, + _ => 0, + }; let source = Box::new(ActiveSource { generation, + refresh_revision, buffer: ActiveSourceBuffer::Ordinary(buffer), playhead_rx, ack: Some(ack), @@ -893,7 +932,11 @@ fn install_source>( }); install_tx .send(if let Some(retire_tx) = retire_tx { - SourceCommand::Refresh { source, retire_tx } + SourceCommand::Refresh { + source, + retire_tx, + requeue_tx: install_tx.clone(), + } } else { SourceCommand::Install(source) }) @@ -943,6 +986,7 @@ fn install_progressive_source>( install_tx .send(SourceCommand::Install(Box::new(ActiveSource { generation, + refresh_revision: 0, buffer: ActiveSourceBuffer::Preparing(buffer), playhead_rx, ack: Some(SourceAcknowledgement::Preparing(request.clone())), @@ -1011,12 +1055,13 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: ControlMsg::Refresh { spec, generation, + revision, retire_tx, } => { if let Err(error) = install_source::( spec, generation, - SourceAcknowledgement::Refresh(retire_tx), + SourceAcknowledgement::Refresh(retire_tx, revision), output_info, false, &source_tx, @@ -1099,6 +1144,7 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: }); } ControlMsg::Retire(source) => drop(source), + ControlMsg::Realign(realign) => realign(), ControlMsg::Shutdown => break, } } @@ -1366,6 +1412,7 @@ mod tests { for (generation, should_replace) in [(7, true), (6, false), (8, false)] { let (mut replacement, _playhead) = source(48_000); replacement.generation = generation; + let replacement_receiver = replacement.playhead_rx.clone(); let ActiveSourceBuffer::Ordinary(buffer) = &mut replacement.buffer else { panic!("expected ordinary source"); }; @@ -1373,10 +1420,11 @@ mod tests { let ActiveSourceBuffer::Ordinary(buffer) = &mut active.as_mut().unwrap().buffer else { panic!("expected ordinary source"); }; - buffer.set_playhead(0.0); + buffer.set_playhead(0.125); tx.send(SourceCommand::Refresh { source: Box::new(replacement), retire_tx: retire_tx.clone(), + requeue_tx: tx.clone(), }) .unwrap(); drain_source_commands(&mut active, &rx); @@ -1386,7 +1434,11 @@ mod tests { )); let actual = active.as_ref().unwrap(); assert_eq!(actual.generation, 7); - assert_eq!(actual.buffer.current_playhead_secs() >= 0.5, should_replace); + assert_eq!( + actual.playhead_rx.same_channel(&replacement_receiver), + should_replace + ); + assert!((actual.buffer.current_playhead_secs() - 0.125).abs() < 1.0 / 48_000.0); } tx.send(SourceCommand::Remove { generation: Some(7), @@ -1397,12 +1449,79 @@ mod tests { tx.send(SourceCommand::Refresh { source: Box::new(replacement), retire_tx: retire_tx.clone(), + requeue_tx: tx.clone(), }) .unwrap(); drain_source_commands(&mut active, &rx); assert!(active.is_none()); } + #[test] + fn enhancement_refresh_realigns_off_callback_and_rejects_superseded_work() { + for (newer_revision, stopped) in [(false, false), (true, false), (false, true)] { + let (mut original, _playhead) = source(48_000); + original.generation = 7; + let ActiveSourceBuffer::Ordinary(buffer) = &mut original.buffer else { + panic!("expected ordinary source"); + }; + buffer.set_playhead(1.0); + let original_receiver = original.playhead_rx.clone(); + let mut active = Some(original); + let (mut replacement, _playhead) = source(48_000); + replacement.generation = 7; + replacement.refresh_revision = 1; + replacement.buffer = ActiveSourceBuffer::Ordinary(PrerenderedAudioBuffer::bounded( + Vec::new(), + MusicTracks::new(), + &ProjectConfiguration::default(), + AudioInfo::new_raw(AudioData::SAMPLE_FORMAT, 48_000, 2), + 0.0, + )); + let replacement_receiver = replacement.playhead_rx.clone(); + let (tx, rx) = std_mpsc::channel(); + let (retire_tx, retire_rx) = std_mpsc::channel(); + tx.send(SourceCommand::Refresh { + source: Box::new(replacement), + retire_tx, + requeue_tx: tx.clone(), + }) + .unwrap(); + drain_source_commands(&mut active, &rx); + let current = active.as_mut().unwrap(); + assert!(current.playhead_rx.same_channel(&original_receiver)); + assert_eq!(current.refresh_revision, 1); + let ControlMsg::Realign(realign) = retire_rx.try_recv().unwrap() else { + panic!("expected deferred audio preparation"); + }; + if newer_revision { + current.refresh_revision = 2; + } + let ActiveSourceBuffer::Ordinary(buffer) = &mut current.buffer else { + panic!("expected ordinary source"); + }; + buffer.set_playhead(1.025); + if stopped { + active = None; + } + realign(); + drain_source_commands(&mut active, &rx); + assert!(matches!( + retire_rx.try_recv().unwrap(), + ControlMsg::Retire(_) + )); + if stopped { + assert!(active.is_none()); + } else { + let current = active.as_ref().unwrap(); + assert_eq!( + current.playhead_rx.same_channel(&replacement_receiver), + !newer_revision + ); + assert!((current.buffer.current_playhead_secs() - 1.025).abs() < 1.0 / 48_000.0); + } + } + } + #[test] fn coalesced_video_updates_preserve_synchronized_audio_position() { for sample_rate in [44_100, 48_000] { diff --git a/crates/editor/src/audio_output/native_tests.rs b/crates/editor/src/audio_output/native_tests.rs index 0798a23c35..7fafc827c2 100644 --- a/crates/editor/src/audio_output/native_tests.rs +++ b/crates/editor/src/audio_output/native_tests.rs @@ -715,12 +715,16 @@ async fn progressive_output_without_a_prefix_never_acknowledges_and_cancels_clea #[test] fn studio_sound_refresh_changes_live_pcm_and_cannot_restart_stopped_audio() { + let original = audible_spec(); + let mut enhanced = audible_spec(); + enhanced.project.audio.improve = true; + let late_refresh = audible_spec(); let (tx, rx) = std_mpsc::channel(); let output = AudioOutput::new_headless(Box::new(move |samples, _| { let mean = samples.iter().map(|sample| sample.abs()).sum::() / samples.len() as f32; let _ = tx.send(mean); })); - let generation = output.play(audible_spec()).unwrap(); + let generation = output.play(original).unwrap(); let wait_for = |predicate: fn(f32) -> bool| { let deadline = Instant::now() + Duration::from_secs(5); loop { @@ -733,13 +737,11 @@ fn studio_sound_refresh_changes_live_pcm_and_cannot_restart_stopped_audio() { } }; wait_for(|mean| mean > 0.1); - let mut enhanced = audible_spec(); - enhanced.project.audio.improve = true; output.refresh_playback(enhanced, generation); wait_for(|mean| mean > 0.005 && mean < 0.04); output.stop_playback(generation); wait_for(|mean| mean == 0.0); - output.refresh_playback(audible_spec(), generation); + output.refresh_playback(late_refresh, generation); for _ in 0..30 { assert_eq!(rx.recv_timeout(Duration::from_secs(2)).unwrap(), 0.0); }