From b1032bdcec14c7de92bb4d266f38583a8383fc16 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 5 May 2026 15:44:54 +0200 Subject: [PATCH] Run RemixAndResample on a background thread The FFI callback thread is OS-audio-scheduled, so per-frame resample DSP there competes with Unity's audio rendering. Move the work to a dedicated per-stream worker thread fed by a bounded BlockingCollection. The FFI handler now just enqueues; the worker drains, resamples, and writes into the existing 200ms ring buffer that absorbs the small extra hop. Co-Authored-By: Claude Opus 4.7 (1M context) --- Runtime/Scripts/AudioStream.cs | 81 +++++++++++++++++++--- Tests/EditMode/MediaStreamLifetimeTests.cs | 10 ++- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index 7915f466..5fc75e26 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Concurrent; +using System.Threading; using UnityEngine; using LiveKit.Internal; using LiveKit.Proto; @@ -40,6 +42,12 @@ public sealed class AudioStream : IDisposable private const int CrossfadeFrames = 128; // ~2.7ms @ 48kHz private int _skipCooldown = 0; + // Resample work runs on a dedicated background thread instead of the FFI callback + // thread, which is OS-audio-scheduled and would otherwise compete with Unity audio. + private readonly BlockingCollection _frameQueue = + new BlockingCollection(new ConcurrentQueue(), boundedCapacity: 50); + private readonly Thread _resampleThread; + /// /// Creates a new audio stream from a remote audio track, attaching it to the /// given in the scene. @@ -73,6 +81,13 @@ public AudioStream(RemoteAudioTrack audioTrack, AudioSource source) // Subscribe to application pause events to handle background/foreground transitions MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause; + + _resampleThread = new Thread(ResampleLoop) + { + IsBackground = true, + Name = "LiveKit AudioStream Resample", + }; + _resampleThread.Start(); } // Called on Unity audio thread @@ -235,7 +250,8 @@ internal void OnApplicationPause(bool pause) } } - // Called on FFI callback thread + // Called on FFI callback thread. Hands the frame off to the worker so the + // OS-audio-scheduled FFI thread doesn't pay for resample DSP. private void OnAudioStreamEvent(AudioStreamEvent e) { if (_disposed) @@ -247,24 +263,58 @@ private void OnAudioStreamEvent(AudioStreamEvent e) if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - using var frame = new AudioFrame(e.FrameReceived.Frame); - - lock (_lock) + var frame = new AudioFrame(e.FrameReceived.Frame); + try { - if (_numChannels == 0) + if (!_frameQueue.IsAddingCompleted && _frameQueue.TryAdd(frame, millisecondsTimeout: 0)) return; + } + catch (InvalidOperationException) + { + // Adding was completed between the IsAddingCompleted check and TryAdd. + } + + // Queue full or shutting down — drop the frame. + Utils.Debug("AudioStream resample queue full; dropping frame"); + frame.Dispose(); + } - unsafe + // Background worker thread. Drains _frameQueue and runs RemixAndResample off the + // FFI callback thread so DSP work doesn't compete with Unity audio rendering. + private void ResampleLoop() + { + try + { + foreach (var frame in _frameQueue.GetConsumingEnumerable()) { - using var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); - if (uFrame != null) + try { - var data = new Span(uFrame.Data.ToPointer(), uFrame.Length); - _buffer?.Write(data); - } + lock (_lock) + { + if (_disposed || _numChannels == 0) + continue; + unsafe + { + using var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); + if (uFrame != null) + { + var data = new Span(uFrame.Data.ToPointer(), uFrame.Length); + _buffer?.Write(data); + } + } + } + } + finally + { + frame.Dispose(); + } } } + catch (Exception ex) + { + Utils.Error($"AudioStream ResampleLoop exited unexpectedly: {ex}"); + } } public void Dispose() @@ -286,6 +336,13 @@ private void Dispose(bool disposing) FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent; MonoBehaviourContext.OnApplicationPauseEvent -= OnApplicationPause; + // Stop the worker before tearing down state it touches under _lock. Joining is only + // safe on the explicit-dispose path; from a finalizer we just signal and let the + // background thread exit on its own. + _frameQueue.CompleteAdding(); + if (disposing && _resampleThread != null && _resampleThread.IsAlive) + _resampleThread.Join(); + lock (_lock) { // Native resources can be released on both the explicit-dispose and finalizer @@ -310,6 +367,8 @@ private void Dispose(bool disposing) Handle.Dispose(); } + _frameQueue.Dispose(); + _disposed = true; } diff --git a/Tests/EditMode/MediaStreamLifetimeTests.cs b/Tests/EditMode/MediaStreamLifetimeTests.cs index 425212d9..1f1e2a2a 100644 --- a/Tests/EditMode/MediaStreamLifetimeTests.cs +++ b/Tests/EditMode/MediaStreamLifetimeTests.cs @@ -122,6 +122,8 @@ public void AudioStream_Dispose_UnsubscribesAndReleasesOwnedResources() StringAssert.Contains("FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent;", source); StringAssert.Contains("_probe.AudioRead -= OnAudioRead;", source); + StringAssert.Contains("_frameQueue.CompleteAdding();", source); + StringAssert.Contains("_resampleThread.Join();", source); StringAssert.Contains("_buffer?.Dispose();", source); StringAssert.Contains("_resampler?.Dispose();", source); StringAssert.Contains("Handle.Dispose();", source); @@ -132,9 +134,11 @@ public void AudioStream_AudioFrames_AreDisposedAfterProcessing() { var source = ReadSource(AudioStreamPaths); - // Both the inbound native frame and the remixed output frame should be scoped so their - // handles are released after each callback rather than accumulating over time. - StringAssert.Contains("using var frame = new AudioFrame(e.FrameReceived.Frame);", source); + // The inbound frame is enqueued on the FFI thread and disposed by the worker + // (in finally) after RemixAndResample. The remixed output frame is still scoped + // with using so handles do not accumulate. + StringAssert.Contains("var frame = new AudioFrame(e.FrameReceived.Frame);", source); + StringAssert.Contains("frame.Dispose();", source); StringAssert.Contains("using var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate);", source); }