corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1913932 - Add an event and a generation id to handle multiple concurrent platform processing params. r=padenot

Browse Source 
As described, this patch adds two things:
A new event where an audio processing track can get notified that new
processing params are about to get applied to the source. This is to handle use
cases involving concurrent processing params, i.e. when two tracks want to apply
different processing params to the same device. A second such track would
otherwise cause the intersection of both tracks' params to get applied before
the first track gets notified async through the result event, possibly causing a
glitch -- if platform params are relaxed before compensated by the track. And,

A generation id of the processing params getting applied to a particular device.
Since the event informing a processing track of the result of applying platform
processing params is async from the new event informing the processing track of
the platform processing params about to get applied, described above, there can
in theory be any number of requests to apply processing params in flight at any
given time. The generation id is needed to distinguish two distinct requests to
apply identical processing params to a given device, where neither of the
requests have signaled their result yet. This allows the processing track to
avoid relaxing the software processing params prematurely, which could cause a
glitch similar to that described for the new event above.

Differential Revision: https://phabricator.services.mozilla.com/D219691
This commit is contained in:
Andreas Pehrson
2024-08-22 09:13:03 +00:00
parent 939b07c4ea
commit 880a44a58f
13 changed files with 396 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
dom/media/DeviceInputTrack.cpp
View File

@@ -316,8 +316,8 @@ bool DeviceInputTrack::HasVoiceInput() const {
return false;
}
cubeb_input_processing_params DeviceInputTrack::RequestedProcessingParams()
const {
AudioInputProcessingParamsRequest
DeviceInputTrack::UpdateRequestedProcessingParams() {
AssertOnGraphThreadOrNotRunning();
Maybe<cubeb_input_processing_params> params;
for (const auto& listener : mListeners) {
@@ -327,7 +327,26 @@ cubeb_input_processing_params DeviceInputTrack::RequestedProcessingParams()
params = Some(listener->RequestedInputProcessingParams(mGraph));
}
}
return params.valueOr(CUBEB_INPUT_PROCESSING_PARAM_NONE);
if (auto p = params.valueOr(CUBEB_INPUT_PROCESSING_PARAM_NONE);
p != mProcessingParamsRequest.mParams) {
mProcessingParamsRequest.mParams = p;
mProcessingParamsRequest.mGeneration += 1;
TRACK_GRAPH_LOG(
"%sNativeInputTrack notifying of setting requested processing params "
"%s (Gen %d)",
(AsNonNativeInputTrack() ? "Non" : ""),
CubebUtils::ProcessingParamsToString(mProcessingParamsRequest.mParams)
.get(),
mProcessingParamsRequest.mGeneration);
NotifySetRequestedProcessingParams(Graph(),
mProcessingParamsRequest.mGeneration,
mProcessingParamsRequest.mParams);
}
return mProcessingParamsRequest;
}
void DeviceInputTrack::DeviceChanged(MediaTrackGraph* aGraph) const {
@@ -340,13 +359,23 @@ void DeviceInputTrack::DeviceChanged(MediaTrackGraph* aGraph) const {
}
}
void DeviceInputTrack::NotifySetRequestedProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams) {
AssertOnGraphThread();
for (const auto& listener : mListeners) {
listener->NotifySetRequestedInputProcessingParams(mGraph, aGeneration,
aRequestedParams);
}
}
void DeviceInputTrack::NotifySetRequestedProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult) {
AssertOnGraphThread();
for (const auto& listener : mListeners) {
listener->NotifySetRequestedInputProcessingParamsResult(
mGraph, aRequestedParams, aResult);
listener->NotifySetRequestedInputProcessingParamsResult(mGraph, aGeneration,
aResult);
}
}
@@ -598,16 +627,17 @@ AudioInputSource::Id NonNativeInputTrack::GenerateSourceId() {
void NonNativeInputTrack::ReevaluateProcessingParams() {
AssertOnGraphThread();
MOZ_ASSERT(mAudioSource);
auto params = RequestedProcessingParams();
if (mRequestedProcessingParams == params) {
auto request = UpdateRequestedProcessingParams();
if (mRequestedProcessingParamsGeneration == request.mGeneration) {
return;
}
mRequestedProcessingParams = params;
auto generation = mRequestedProcessingParamsGeneration = request.mGeneration;
auto params = request.mParams;
using Promise = AudioInputSource::SetRequestedProcessingParamsPromise;
mAudioSource->SetRequestedProcessingParams(params)->Then(
GetMainThreadSerialEventTarget(), __func__,
[this, self = RefPtr(this),
params](Promise::ResolveOrRejectValue&& aValue) {
generation](Promise::ResolveOrRejectValue&& aValue) {
if (IsDestroyed()) {
return;
}
@@ -617,10 +647,11 @@ void NonNativeInputTrack::ReevaluateProcessingParams() {
}
return Err(aValue.RejectValue());
})();
QueueControlMessageWithNoShutdown(
[this, self = RefPtr(this), params, result = std::move(result)] {
NotifySetRequestedProcessingParamsResult(Graph(), params, result);
});
QueueControlMessageWithNoShutdown([this, self = RefPtr(this),
generation,
result = std::move(result)] {
NotifySetRequestedProcessingParamsResult(Graph(), generation, result);
});
});
}

19
dom/media/DeviceInputTrack.h
View File

@@ -177,13 +177,22 @@ class DeviceInputTrack : public ProcessedMediaTrack {
// Query audio settings from its users.
uint32_t MaxRequestedInputChannels() const;
bool HasVoiceInput() const;
// Query for the aggregate processing params from all users.
cubeb_input_processing_params RequestedProcessingParams() const;
// Query for the aggregated form of processing params from all consumers. If
// different from the previous call, the generation is updated and listeners
// notified that new processing params are being requested. The caller is
// responsible for performing the request.
[[nodiscard]] AudioInputProcessingParamsRequest
UpdateRequestedProcessingParams();
// Signal to listeners that the requested platform processing params is about
// to change.
void NotifySetRequestedProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams);
// Handle the result of an async operation to set processing params on a cubeb
// stream. If the operation failed, signal this to listeners and then disable
// processing. If the operation succeeded, directly signal this to listeners.
void NotifySetRequestedProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult);
// Deliver notification to its users.
void DeviceChanged(MediaTrackGraph* aGraph) const;
@@ -214,6 +223,7 @@ class DeviceInputTrack : public ProcessedMediaTrack {
// Only accessed on the graph thread.
nsTArray<RefPtr<AudioDataListener>> mListeners;
AudioInputProcessingParamsRequest mProcessingParamsRequest;
};
class NativeInputTrack final : public DeviceInputTrack {
@@ -281,8 +291,7 @@ class NonNativeInputTrack final : public DeviceInputTrack {
// Graph thread only.
RefPtr<AudioInputSource> mAudioSource;
AudioInputSource::Id mSourceIdNumber;
cubeb_input_processing_params mRequestedProcessingParams =
CUBEB_INPUT_PROCESSING_PARAM_NONE;
int mRequestedProcessingParamsGeneration{};
#ifdef DEBUG
// Graph thread only.

81
dom/media/GraphDriver.cpp
View File

@@ -343,8 +343,7 @@ class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
}
void NotifySetRequestedInputProcessingParamsResult(
AudioCallbackDriver* aDriver,
cubeb_input_processing_params aRequestedParams,
AudioCallbackDriver* aDriver, int aGeneration,
Result<cubeb_input_processing_params, int>&& aResult) override {
MOZ_CRASH(
"Unexpected processing params result from fallback SystemClockDriver");
@@ -465,7 +464,7 @@ AudioCallbackDriver::AudioCallbackDriver(
uint32_t aSampleRate, uint32_t aOutputChannelCount,
uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType,
cubeb_input_processing_params aRequestedInputProcessingParams)
Maybe<AudioInputProcessingParamsRequest> aRequestedInputProcessingParams)
: GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
mOutputChannelCount(aOutputChannelCount),
mInputChannelCount(aInputChannelCount),
@@ -473,7 +472,8 @@ AudioCallbackDriver::AudioCallbackDriver(
mInputDeviceID(aInputDeviceID),
mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
mCubebOperationThread(CreateTaskQueue()),
mRequestedInputProcessingParams(aRequestedInputProcessingParams),
mInputProcessingRequest(aRequestedInputProcessingParams.valueOr(
AudioInputProcessingParamsRequest{})),
mAudioThreadId(ProfilerThreadId{}),
mAudioThreadIdInCb(std::thread::id()),
mFallback("AudioCallbackDriver::mFallback"),
@@ -491,10 +491,11 @@ AudioCallbackDriver::AudioCallbackDriver(
media_getusermedia_microphone_prefer_voice_stream_with_processing_enabled()) {
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver %p ctor - using VOICE and requesting input "
"processing params %s.",
"processing params %s (Gen %d).",
Graph(), this,
CubebUtils::ProcessingParamsToString(aRequestedInputProcessingParams)
.get()));
CubebUtils::ProcessingParamsToString(mInputProcessingRequest.mParams)
.get(),
mInputProcessingRequest.mGeneration));
mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
CubebUtils::SetInCommunication(true);
} else {
@@ -680,7 +681,7 @@ void AudioCallbackDriver::Init(const nsCString& aStreamName) {
#endif
if (inputWanted && InputDevicePreference() == AudioInputType::Voice) {
SetInputProcessingParams(mRequestedInputProcessingParams);
SetInputProcessingParams(mInputProcessingRequest);
}
cubeb_stream_register_device_changed_callback(
@@ -1391,32 +1392,34 @@ void AudioCallbackDriver::MaybeStartAudioStream() {
Start();
}
cubeb_input_processing_params
const AudioInputProcessingParamsRequest&
AudioCallbackDriver::RequestedInputProcessingParams() const {
MOZ_ASSERT(InIteration());
return mRequestedInputProcessingParams;
return mInputProcessingRequest;
}
void AudioCallbackDriver::SetRequestedInputProcessingParams(
cubeb_input_processing_params aParams) {
void AudioCallbackDriver::RequestInputProcessingParams(
AudioInputProcessingParamsRequest aRequest) {
MOZ_ASSERT(InIteration());
if (mRequestedInputProcessingParams == aParams) {
return;
}
MOZ_ASSERT(aRequest.mGeneration > mInputProcessingRequest.mGeneration);
MOZ_ASSERT(aRequest.mParams != mInputProcessingRequest.mParams);
LOG(LogLevel::Info,
("AudioCallbackDriver %p, Input processing params %s requested.", this,
CubebUtils::ProcessingParamsToString(aParams).get()));
mRequestedInputProcessingParams = aParams;
("AudioCallbackDriver %p, Input processing params %s (Gen %d) requested.",
this, CubebUtils::ProcessingParamsToString(aRequest.mParams).get(),
aRequest.mGeneration));
mInputProcessingRequest = aRequest;
MOZ_ALWAYS_SUCCEEDS(mCubebOperationThread->Dispatch(
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), aParams] {
SetInputProcessingParams(aParams);
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), aRequest] {
SetInputProcessingParams(aRequest);
})));
}
void AudioCallbackDriver::SetInputProcessingParams(
cubeb_input_processing_params aParams) {
AudioInputProcessingParamsRequest aRequest) {
MOZ_ASSERT(OnCubebOperationThread());
auto requested = aParams;
const auto requested = aRequest.mParams;
auto params = aRequest.mParams;
const auto generation = aRequest.mGeneration;
auto result = ([&]() -> Maybe<Result<cubeb_input_processing_params, int>> {
// This function decides how to handle the request.
// Returning Nothing() does nothing, because either
@@ -1445,50 +1448,50 @@ void AudioCallbackDriver::SetInputProcessingParams(
("AudioCallbackDriver %p, no supported processing params", this));
return Some(Err(CUBEB_ERROR_NOT_SUPPORTED));
}
aParams &= supported;
params &= supported;
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, requested processing params %s reduced to %s "
"by supported params %s",
("AudioCallbackDriver %p, requested processing params %s (gen %d) "
"reduced to %s by supported params %s",
this, CubebUtils::ProcessingParamsToString(requested).get(),
CubebUtils::ProcessingParamsToString(aParams).get(),
generation, CubebUtils::ProcessingParamsToString(params).get(),
CubebUtils::ProcessingParamsToString(supported).get()));
if (aParams == mConfiguredInputProcessingParams) {
if (params == mConfiguredInputProcessingParams) {
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, no change in processing params %s. Not "
"attempting reconfiguration.",
this, CubebUtils::ProcessingParamsToString(aParams).get()));
return Some(aParams);
this, CubebUtils::ProcessingParamsToString(params).get()));
return Some(params);
}
mConfiguredInputProcessingParams = aParams;
r = cubeb_stream_set_input_processing_params(mAudioStream, aParams);
mConfiguredInputProcessingParams = params;
r = cubeb_stream_set_input_processing_params(mAudioStream, params);
if (r == CUBEB_OK) {
LOG(LogLevel::Info,
("AudioCallbackDriver %p, input processing params set to %s", this,
CubebUtils::ProcessingParamsToString(aParams).get()));
return Some(aParams);
CubebUtils::ProcessingParamsToString(params).get()));
return Some(params);
}
LOG(LogLevel::Info,
("AudioCallbackDriver %p, failed setting input processing params to "
"%s. r=%d",
this, CubebUtils::ProcessingParamsToString(aParams).get(), r));
this, CubebUtils::ProcessingParamsToString(params).get(), r));
return Some(Err(r));
})();
if (!result) {
return;
}
MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), requested,
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), generation,
result = result.extract()]() mutable {
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, Notifying of input processing params %s. "
"r=%d",
("AudioCallbackDriver %p, Notifying of input processing params %s "
"(Gen %d). r=%d",
this,
CubebUtils::ProcessingParamsToString(
result.unwrapOr(CUBEB_INPUT_PROCESSING_PARAM_NONE))
.get(),
result.isErr() ? result.inspectErr() : CUBEB_OK));
generation, result.isErr() ? result.inspectErr() : CUBEB_OK));
mGraphInterface->NotifySetRequestedInputProcessingParamsResult(
this, requested, std::move(result));
this, generation, std::move(result));
})));
}

32
dom/media/GraphDriver.h
View File

@@ -180,10 +180,10 @@ struct GraphInterface : public nsISupports {
TrackRate aRate, uint32_t aChannels,
uint32_t aAlreadyBuffered) = 0;
/* Called on the main thread after an AudioCallbackDriver has attempted an
* operation to set aRequestedParams on the cubeb stream. */
* operation to set the processing params matching aGeneration on the cubeb
* stream. */
virtual void NotifySetRequestedInputProcessingParamsResult(
AudioCallbackDriver* aDriver,
cubeb_input_processing_params aRequestedParams,
AudioCallbackDriver* aDriver, int aGeneration,
Result<cubeb_input_processing_params, int>&& aResult) = 0;
/* Called every time there are changes to input/output audio devices like
* plug/unplug etc. This can be called on any thread, and posts a message to
@@ -523,6 +523,11 @@ class OfflineClockDriver : public ThreadedDriver {
enum class AudioInputType { Unknown, Voice };
struct AudioInputProcessingParamsRequest {
int mGeneration{};
cubeb_input_processing_params mParams{};
};
/**
* This is a graph driver that is based on callback functions called by the
* audio api. This ensures minimal audio latency, because it means there is no
@@ -558,7 +563,7 @@ class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver {
uint32_t aSampleRate, uint32_t aOutputChannelCount,
uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType,
cubeb_input_processing_params aRequestedInputProcessingParams);
Maybe<AudioInputProcessingParamsRequest> aRequestedInputProcessingParams);
void Start() override;
MOZ_CAN_RUN_SCRIPT void Shutdown() override;
@@ -610,13 +615,14 @@ class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver {
return AudioInputType::Unknown;
}
/* Get the input processing params requested from this driver, so that an
* external caller can decide whether it is necessary to call the setter,
* since it may allocate or dispatch. */
cubeb_input_processing_params RequestedInputProcessingParams() const;
/* Get the latest input processing params request from this driver, so
* that an external caller can decide whether it is necessary to call the
* setter, since it may allocate or dispatch. */
const AudioInputProcessingParamsRequest& RequestedInputProcessingParams()
const;
/* Set the input processing params requested from this driver. */
void SetRequestedInputProcessingParams(cubeb_input_processing_params aParams);
void RequestInputProcessingParams(AudioInputProcessingParamsRequest);
std::thread::id ThreadId() const { return mAudioThreadIdInCb.load(); }
@@ -665,7 +671,7 @@ class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver {
void Stop();
/* After the requested input processing params has changed, this applies them
* on the cubeb stream. */
void SetInputProcessingParams(cubeb_input_processing_params aParams);
void SetInputProcessingParams(AudioInputProcessingParamsRequest aRequest);
/* Calls FallbackToSystemClockDriver() if in FallbackDriverState::None.
* Returns Ok(true) if the fallback driver was started, or the old
* FallbackDriverState in an Err otherwise. */
@@ -739,9 +745,9 @@ class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver {
* supported processing params. Cubeb operation thread only. */
cubeb_input_processing_params mConfiguredInputProcessingParams =
CUBEB_INPUT_PROCESSING_PARAM_NONE;
/* The input processing params requested from this audio driver. Once started,
* audio callback thread only. */
cubeb_input_processing_params mRequestedInputProcessingParams;
/* The input processing params and generation requested from this audio
* driver. Once started, audio callback thread only. */
AudioInputProcessingParamsRequest mInputProcessingRequest;
/* Contains the id of the audio thread, from profiler_current_thread_id. */
std::atomic<ProfilerThreadId> mAudioThreadId;
/* This allows implementing AutoInCallback. This is equal to the current

54
dom/media/MediaTrackGraph.cpp
View File

@@ -437,8 +437,10 @@ void MediaTrackGraphImpl::CheckDriver() {
CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
uint32_t inputChannelCount = AudioInputChannelCount(inputDevice);
AudioInputType inputPreference = AudioInputDevicePreference(inputDevice);
cubeb_input_processing_params inputProcessingParams =
RequestedAudioInputProcessingParams(inputDevice);
Maybe<AudioInputProcessingParamsRequest> processingRequest =
ToMaybeRef(native).map([](auto& native) {
return native.UpdateRequestedProcessingParams();
});
uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount();
if (!audioCallbackDriver) {
@@ -446,12 +448,17 @@ void MediaTrackGraphImpl::CheckDriver() {
AudioCallbackDriver* driver = new AudioCallbackDriver(
this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
inputPreference, inputProcessingParams);
inputPreference, processingRequest);
SwitchAtNextIteration(driver);
}
return;
}
bool needInputProcessingParamUpdate =
processingRequest &&
processingRequest->mGeneration !=
audioCallbackDriver->RequestedInputProcessingParams().mGeneration;
// Check if this graph should switch to a different number of output channels.
// Generally, a driver switch is explicitly made by an event (e.g., setting
// the AudioDestinationNode channelCount), but if an HTMLMediaElement is
@@ -459,16 +466,23 @@ void MediaTrackGraphImpl::CheckDriver() {
// of the media determines how many channels to output, and it can change
// dynamically.
if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
if (needInputProcessingParamUpdate) {
needInputProcessingParamUpdate = false;
}
AudioCallbackDriver* driver = new AudioCallbackDriver(
this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
inputPreference, inputProcessingParams);
inputPreference, processingRequest);
SwitchAtNextIteration(driver);
}
if (native) {
audioCallbackDriver->SetRequestedInputProcessingParams(
inputProcessingParams);
if (needInputProcessingParamUpdate) {
needInputProcessingParamUpdate = false;
LOG(LogLevel::Debug,
("%p: Setting on the fly requested processing params %s (Gen %d)", this,
CubebUtils::ProcessingParamsToString(processingRequest->mParams).get(),
processingRequest->mGeneration));
audioCallbackDriver->RequestInputProcessingParams(*processingRequest);
}
}
@@ -770,7 +784,7 @@ void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId),
aTrack->RequestedProcessingParams());
Some(aTrack->UpdateRequestedProcessingParams()));
LOG(LogLevel::Debug,
("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
this, driver));
@@ -843,7 +857,7 @@ void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
nullptr, AudioInputDevicePreference(aTrack->mDeviceId),
aTrack->RequestedProcessingParams());
Some(aTrack->UpdateRequestedProcessingParams()));
SwitchAtNextIteration(driver);
} else if (CurrentDriver()->AsAudioCallbackDriver()) {
LOG(LogLevel::Debug,
@@ -939,8 +953,7 @@ void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
}
void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult(
AudioCallbackDriver* aDriver,
cubeb_input_processing_params aRequestedParams,
AudioCallbackDriver* aDriver, int aGeneration,
Result<cubeb_input_processing_params, int>&& aResult) {
MOZ_ASSERT(NS_IsMainThread());
NativeInputTrack* native =
@@ -949,7 +962,7 @@ void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult(
return;
}
QueueControlMessageWithNoShutdown([this, self = RefPtr(this),
driver = RefPtr(aDriver), aRequestedParams,
driver = RefPtr(aDriver), aGeneration,
result = std::move(aResult)]() mutable {
NativeInputTrack* native =
mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
@@ -959,8 +972,7 @@ void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult(
if (driver != mDriver) {
return;
}
native->NotifySetRequestedProcessingParamsResult(this, aRequestedParams,
result);
native->NotifySetRequestedProcessingParamsResult(this, aGeneration, result);
});
}
@@ -1142,7 +1154,8 @@ void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
AudioCallbackDriver* newDriver = new AudioCallbackDriver(
this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
AudioInputChannelCount(aID), PrimaryOutputDeviceID(), aID,
AudioInputDevicePreference(aID), track->RequestedProcessingParams());
AudioInputDevicePreference(aID),
Some(track->UpdateRequestedProcessingParams()));
SwitchAtNextIteration(newDriver);
}
}
@@ -3487,7 +3500,7 @@ void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested,
// for the input channel.
mDriver = new AudioCallbackDriver(
this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(),
nullptr, AudioInputType::Unknown, CUBEB_INPUT_PROCESSING_PARAM_NONE);
nullptr, AudioInputType::Unknown, Nothing());
} else {
mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
}
@@ -4283,15 +4296,6 @@ AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
: AudioInputType::Unknown;
}
cubeb_input_processing_params
MediaTrackGraphImpl::RequestedAudioInputProcessingParams(
CubebUtils::AudioDeviceID aID) {
MOZ_ASSERT(OnGraphThreadOrNotRunning());
DeviceInputTrack* t =
mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
return t ? t->RequestedProcessingParams() : CUBEB_INPUT_PROCESSING_PARAM_NONE;
}
void MediaTrackGraphImpl::SetNewNativeInput() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());

13
dom/media/MediaTrackGraph.h
View File

@@ -137,11 +137,20 @@ class AudioDataListenerInterface {
virtual void Disconnect(MediaTrackGraph* aGraph) = 0;
/**
* Called after an attempt to set the input processing params on the
* Called sync when attempting to set the input processing params on the
* underlying input track. Note that when multiple listeners request distinct
* parameters, aRequestedParams is the aggregated form of those parameters.
*/
virtual void NotifySetRequestedInputProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams) = 0;
/**
* Called async after an attempt to set the input processing params on the
* underlying input track.
*/
virtual void NotifySetRequestedInputProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult) = 0;
};

10
dom/media/MediaTrackGraphImpl.h
View File

@@ -512,8 +512,7 @@ class MediaTrackGraphImpl : public MediaTrackGraph,
/* Called on the main thread after an AudioCallbackDriver has attempted an
* operation to set aRequestedParams on the cubeb stream. */
void NotifySetRequestedInputProcessingParamsResult(
AudioCallbackDriver* aDriver,
cubeb_input_processing_params aRequestedParams,
AudioCallbackDriver* aDriver, int aGeneration,
Result<cubeb_input_processing_params, int>&& aResult) override;
/* Called every time there are changes to input/output audio devices like
* plug/unplug etc. This can be called on any thread, and posts a message to
@@ -592,13 +591,6 @@ class MediaTrackGraphImpl : public MediaTrackGraph,
AudioInputType AudioInputDevicePreference(CubebUtils::AudioDeviceID aID);
/**
* The input processing params requested for any processing tracks tied to the
* input device with id aID.
*/
cubeb_input_processing_params RequestedAudioInputProcessingParams(
CubebUtils::AudioDeviceID aID);
double MediaTimeToSeconds(GraphTime aTime) const {
NS_ASSERTION(aTime > -TRACK_TIME_MAX && aTime <= TRACK_TIME_MAX,
"Bad time");

122
dom/media/gtest/TestAudioCallbackDriver.cpp
View File

@@ -37,7 +37,7 @@ class MockGraphInterface : public GraphInterface {
MOCK_METHOD(void, NotifyInputData,
(const AudioDataValue*, size_t, TrackRate, uint32_t, uint32_t));
MOCK_METHOD(void, NotifySetRequestedInputProcessingParamsResult,
(AudioCallbackDriver*, cubeb_input_processing_params,
(AudioCallbackDriver*, int,
(Result<cubeb_input_processing_params, int>&&)));
MOCK_METHOD(void, DeviceChanged, ());
#ifdef DEBUG
@@ -122,9 +122,10 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
auto graph = MakeRefPtr<NiceMock<MockGraphInterface>>(rate);
EXPECT_CALL(*graph, NotifyInputStopped).Times(0);
driver = MakeRefPtr<AudioCallbackDriver>(graph, nullptr, rate, 2, 0, nullptr,
nullptr, AudioInputType::Unknown,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 0, nullptr, nullptr, AudioInputType::Unknown,
Some<AudioInputProcessingParamsRequest>(
{0, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
@@ -182,9 +183,10 @@ void TestSlowStart(const TrackRate aRate) MOZ_CAN_RUN_SCRIPT_BOUNDARY {
inputFrameCount += aFrames;
});
driver = MakeRefPtr<AudioCallbackDriver>(graph, nullptr, aRate, 2, 2, nullptr,
(void*)1, AudioInputType::Voice,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, aRate, 2, 2, nullptr, (void*)1, AudioInputType::Voice,
Some<AudioInputProcessingParamsRequest>(
{0, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
@@ -271,10 +273,12 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual);
CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext());
int generation = 99;
auto graph = MakeRefPtr<MockGraphInterface>(rate);
auto driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 1, nullptr, (void*)1, AudioInputType::Voice,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
Some<AudioInputProcessingParamsRequest>(
{generation, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
@@ -291,8 +295,7 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
Result<cubeb_input_processing_params, int> expected =
Err(CUBEB_ERROR_NOT_SUPPORTED);
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(), CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(expected))));
driver.get(), generation, Eq(std::ref(expected))));
graph->SetCurrentDriver(driver);
graph->SetEnsureNextIteration(true);
@@ -410,13 +413,15 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
auto graph = MakeRefPtr<MockGraphInterface>(rate);
auto driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 1, nullptr, (void*)1, AudioInputType::Voice,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
Some<AudioInputProcessingParamsRequest>(
{99, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
auto newDriver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 1, nullptr, (void*)1, AudioInputType::Voice,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
Some<AudioInputProcessingParamsRequest>(
{99, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(newDriver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(newDriver->IsStarted()) << "Verify thread is not started";
@@ -435,11 +440,9 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
Result<cubeb_input_processing_params, int> expected =
Err(CUBEB_ERROR_NOT_SUPPORTED);
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(), CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(expected))));
driver.get(), 99, Eq(std::ref(expected))));
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
newDriver.get(), CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(expected))));
newDriver.get(), 99, Eq(std::ref(expected))));
graph->SetCurrentDriver(driver);
graph->SetEnsureNextIteration(true);
@@ -566,7 +569,8 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
}
void TestInputProcessingOnStart(
MockCubeb* aCubeb, cubeb_input_processing_params aRequested,
MockCubeb* aCubeb, int aGeneration,
cubeb_input_processing_params aRequested,
const Result<cubeb_input_processing_params, int>& aExpected)
MOZ_CAN_RUN_SCRIPT_BOUNDARY {
const TrackRate rate = 44100;
@@ -574,7 +578,7 @@ void TestInputProcessingOnStart(
auto graph = MakeRefPtr<NiceMock<MockGraphInterface>>(rate);
auto driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 1, nullptr, nullptr, AudioInputType::Voice,
aRequested);
Some<AudioInputProcessingParamsRequest>({aGeneration, aRequested}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
@@ -587,7 +591,7 @@ void TestInputProcessingOnStart(
bool notified = false;
EXPECT_CALL(*graph, NotifyInputStopped).Times(0);
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(), aRequested, Eq(std::ref(aExpected))))
driver.get(), aGeneration, Eq(std::ref(aExpected))))
.WillOnce([&] { notified = true; });
graph->SetCurrentDriver(driver);
@@ -638,30 +642,31 @@ TEST(TestAudioCallbackDriver, InputProcessingOnStart)
// Not supported by backend.
cubeb->SetSupportedInputProcessingParams(CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_ERROR_NOT_SUPPORTED);
TestInputProcessingOnStart(cubeb,
TestInputProcessingOnStart(cubeb, 1,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Err(CUBEB_ERROR_NOT_SUPPORTED));
// Not supported by params.
cubeb->SetSupportedInputProcessingParams(CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_OK);
TestInputProcessingOnStart(cubeb,
TestInputProcessingOnStart(cubeb, 2,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
// Successful all.
cubeb->SetSupportedInputProcessingParams(allParams, CUBEB_OK);
TestInputProcessingOnStart(cubeb, allParams, allParams);
TestInputProcessingOnStart(cubeb, 3, allParams, allParams);
// Successful partial.
TestInputProcessingOnStart(cubeb,
TestInputProcessingOnStart(cubeb, 4,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
// Not supported by stream.
cubeb->SetInputProcessingApplyRv(CUBEB_ERROR);
TestInputProcessingOnStart(
cubeb, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION, Err(CUBEB_ERROR));
TestInputProcessingOnStart(cubeb, 5,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Err(CUBEB_ERROR));
}
TEST(TestAudioCallbackDriver, InputProcessingWhileRunning)
@@ -674,13 +679,14 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
CUBEB_INPUT_PROCESSING_PARAM_VOICE_ISOLATION;
constexpr int applyError = 99;
int numNotifications = 0;
const auto signal = [&]() mutable {
int currentGeneration = 0;
const auto signal = [&](auto aDriver, auto aGeneration,
auto&& aResult) mutable {
MOZ_ASSERT(NS_IsMainThread());
++numNotifications;
currentGeneration = aGeneration;
};
const auto waitForSignal = [&](int aNotification) {
while (numNotifications < aNotification) {
const auto waitForSignal = [&](int aGeneration) {
while (currentGeneration != aGeneration) {
NS_ProcessNextEvent();
}
};
@@ -690,7 +696,8 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
auto graph = MakeRefPtr<NiceMock<MockGraphInterface>>(rate);
auto driver = MakeRefPtr<AudioCallbackDriver>(
graph, nullptr, rate, 2, 1, nullptr, nullptr, AudioInputType::Voice,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
Some<AudioInputProcessingParamsRequest>(
{100, CUBEB_INPUT_PROCESSING_PARAM_NONE}));
EXPECT_FALSE(driver->ThreadRunning()) << "Verify thread is not running";
EXPECT_FALSE(driver->IsStarted()) << "Verify thread is not started";
@@ -709,37 +716,30 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
InSequence s;
// Notified on start.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(), CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(notSupportedResult))))
EXPECT_CALL(*graph,
NotifySetRequestedInputProcessingParamsResult(
driver.get(), 100, Eq(std::ref(notSupportedResult))))
.WillOnce(signal);
// Not supported by backend.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(),
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(notSupportedResult))))
EXPECT_CALL(*graph,
NotifySetRequestedInputProcessingParamsResult(
driver.get(), 101, Eq(std::ref(notSupportedResult))))
.WillOnce(signal);
// Not supported by params.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(),
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(noneResult))))
driver.get(), 102, Eq(std::ref(noneResult))))
.WillOnce(signal);
// Successful all.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(), allParams, Eq(std::ref(allResult))))
driver.get(), 103, Eq(std::ref(allResult))))
.WillOnce(signal);
// Successful partial.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(),
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(aecResult))))
driver.get(), 104, Eq(std::ref(aecResult))))
.WillOnce(signal);
// Not supported by stream.
EXPECT_CALL(*graph, NotifySetRequestedInputProcessingParamsResult(
driver.get(),
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(applyErrorResult))))
driver.get(), 105, Eq(std::ref(applyErrorResult))))
.WillOnce(signal);
}
@@ -750,12 +750,12 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
});
#endif
const auto setParams = [&](cubeb_input_processing_params aParams) {
const auto setParams = [&](int aGen, cubeb_input_processing_params aParams) {
{
#ifdef DEBUG
AutoSetter as(inGraphIteration, true);
#endif
driver->SetRequestedInputProcessingParams(aParams);
driver->RequestInputProcessingParams({aGen, aParams});
}
};
@@ -783,33 +783,33 @@ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
waitForSignal(1);
waitForSignal(100);
// Not supported by backend.
cubeb->SetSupportedInputProcessingParams(CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_ERROR_NOT_SUPPORTED);
setParams(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
waitForSignal(2);
setParams(101, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
waitForSignal(101);
// Not supported by params.
cubeb->SetSupportedInputProcessingParams(CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_OK);
setParams(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
waitForSignal(3);
setParams(102, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
waitForSignal(102);
// Successful all.
cubeb->SetSupportedInputProcessingParams(allParams, CUBEB_OK);
setParams(allParams);
waitForSignal(4);
setParams(103, allParams);
waitForSignal(103);
// Successful partial.
setParams(CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
waitForSignal(5);
setParams(104, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
waitForSignal(104);
// Not supported by stream.
cubeb->SetInputProcessingApplyRv(applyError);
setParams(CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
waitForSignal(6);
setParams(105, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
waitForSignal(105);
// This will block untill all events have been executed.
MOZ_KnownLive(driver)->Shutdown();

108
dom/media/gtest/TestAudioInputProcessing.cpp
View File

@@ -535,17 +535,32 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// No other constraint requests present.
aip->NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// Platform processing params successfully applied.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
// Turns off the equivalent APM config.
EXPECT_EQ(aip->AppliedConfig(graph), webrtc::AudioProcessing::Config());
EXPECT_TRUE(aip->IsPassThrough(graph));
// Request for a response that comes back out-of-order later.
aip->NotifySetRequestedInputProcessingParams(
graph, 2, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
// Simulate an error after a driver switch.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION, Err(CUBEB_ERROR));
aip->NotifySetRequestedInputProcessingParams(
graph, 3, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
// Requesting the same config that is already applied; does nothing.
EXPECT_EQ(aip->AppliedConfig(graph), webrtc::AudioProcessing::Config());
EXPECT_TRUE(aip->IsPassThrough(graph));
// Error notification.
aip->NotifySetRequestedInputProcessingParamsResult(graph, 3,
Err(CUBEB_ERROR));
// The APM config is turned back on, and platform processing is requested to
// be turned off.
EXPECT_EQ(aip->RequestedInputProcessingParams(graph),
@@ -553,10 +568,15 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// The request for turning platform processing off.
aip->NotifySetRequestedInputProcessingParams(
graph, 4, CUBEB_INPUT_PROCESSING_PARAM_NONE);
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// Pretend there was a response for an old request.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
graph, 2, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
// It does nothing since we are requesting NONE now.
EXPECT_EQ(aip->RequestedInputProcessingParams(graph),
CUBEB_INPUT_PROCESSING_PARAM_NONE);
@@ -565,8 +585,7 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
// Turn it off as requested.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
graph, 4, CUBEB_INPUT_PROCESSING_PARAM_NONE);
EXPECT_EQ(aip->RequestedInputProcessingParams(graph),
CUBEB_INPUT_PROCESSING_PARAM_NONE);
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
@@ -580,22 +599,29 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// The request doesn't change anything.
aip->NotifySetRequestedInputProcessingParams(
graph, 5,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// Only noise suppression was supported in the platform.
aip->NotifySetRequestedInputProcessingParamsResult(
graph,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
graph, 5, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
// In the APM only echo cancellation is applied.
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// Test error for partial support.
aip->NotifySetRequestedInputProcessingParamsResult(
graph,
aip->NotifySetRequestedInputProcessingParams(
graph, 6,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Err(CUBEB_ERROR));
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
EXPECT_EQ(aip->AppliedConfig(graph), echoOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
aip->NotifySetRequestedInputProcessingParamsResult(graph, 6,
Err(CUBEB_ERROR));
// The full config is applied in the APM, and NONE is requested.
EXPECT_EQ(aip->RequestedInputProcessingParams(graph),
CUBEB_INPUT_PROCESSING_PARAM_NONE);
@@ -609,11 +635,16 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// Request.
aip->NotifySetRequestedInputProcessingParams(
graph, 7,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// It succeeded.
aip->NotifySetRequestedInputProcessingParamsResult(
graph,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
graph, 7,
static_cast<cubeb_input_processing_params>(
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
@@ -627,11 +658,15 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
// Simulate that another concurrent request was made, i.e. two tracks are
// using the same device with different processing params, where the
// intersection of processing params is NONE.
aip->NotifySetRequestedInputProcessingParams(
graph, 8, CUBEB_INPUT_PROCESSING_PARAM_NONE);
// The full config is applied in the APM.
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// The result succeeds, leading to no change since sw processing is already
// applied.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
CUBEB_INPUT_PROCESSING_PARAM_NONE);
// The full config is applied in the APM, and NONE is requested.
graph, 8, CUBEB_INPUT_PROCESSING_PARAM_NONE);
EXPECT_EQ(aip->RequestedInputProcessingParams(graph),
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
@@ -639,11 +674,17 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
EXPECT_FALSE(aip->IsPassThrough(graph));
// The other concurrent request goes away.
// The result succeeds, removing the sw processing config.
aip->NotifySetRequestedInputProcessingParamsResult(
graph,
aip->NotifySetRequestedInputProcessingParams(
graph, 9,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION);
// The full config is still applied in the APM.
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
// The result succeeds, leading to no change since sw processing is already
// applied.
aip->NotifySetRequestedInputProcessingParamsResult(
graph, 9,
static_cast<cubeb_input_processing_params>(
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
@@ -653,14 +694,15 @@ TEST(TestAudioInputProcessing, PlatformProcessing)
EXPECT_EQ(aip->AppliedConfig(graph), webrtc::AudioProcessing::Config());
EXPECT_TRUE(aip->IsPassThrough(graph));
// The applied config (AEC, NS) is adapted to the subset applied in the
// platform (AEC).
// Changing input track resets the processing params generation. The applied
// config (AEC, NS) is adapted to the subset applied in the platform (AEC).
aip->Disconnect(graph);
aip->NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
EXPECT_EQ(aip->AppliedConfig(graph), echoNoiseConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));
aip->NotifySetRequestedInputProcessingParamsResult(
graph,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION |
CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION);
EXPECT_EQ(aip->AppliedConfig(graph), noiseOnlyConfig);
EXPECT_FALSE(aip->IsPassThrough(graph));

88
dom/media/gtest/TestAudioTrackGraph.cpp
View File

@@ -158,8 +158,10 @@ class MockAudioDataListener : public AudioDataListener {
MOCK_METHOD(bool, IsVoiceInput, (MediaTrackGraph*), (const));
MOCK_METHOD(void, DeviceChanged, (MediaTrackGraph*));
MOCK_METHOD(void, Disconnect, (MediaTrackGraph*));
MOCK_METHOD(void, NotifySetRequestedInputProcessingParams,
(MediaTrackGraph*, int, cubeb_input_processing_params));
MOCK_METHOD(void, NotifySetRequestedInputProcessingParamsResult,
(MediaTrackGraph*, cubeb_input_processing_params,
(MediaTrackGraph*, int,
(const Result<cubeb_input_processing_params, int>&)));
};
} // namespace
@@ -2872,43 +2874,48 @@ TEST(TestAudioTrackGraph, PlatformProcessing)
InSequence s;
// On first driver start.
EXPECT_CALL(*listener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(echoResult))))
NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, 1, Eq(std::ref(echoResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After requesting something else.
EXPECT_CALL(*listener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(noiseResult))))
NotifySetRequestedInputProcessingParams(
graph, 2, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, 2, Eq(std::ref(noiseResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After error request.
EXPECT_CALL(*listener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(errorResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After requesting None.
NotifySetRequestedInputProcessingParams(
graph, 3, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(noneResult))))
graph, 3, Eq(std::ref(errorResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After requesting None, because of the previous error.
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParams(
graph, 4, CUBEB_INPUT_PROCESSING_PARAM_NONE));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, 4, Eq(std::ref(noneResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After driver switch.
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(noneResult))))
graph, 4, Eq(std::ref(noneResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After requesting something not supported.
EXPECT_CALL(*listener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(noneResult))))
NotifySetRequestedInputProcessingParams(
graph, 5, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, 5, Eq(std::ref(noneResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After requesting something with backend not supporting processing params.
EXPECT_CALL(*listener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(notSupportedResult))))
NotifySetRequestedInputProcessingParams(
graph, 6, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
EXPECT_CALL(*listener, NotifySetRequestedInputProcessingParamsResult(
graph, 6, Eq(std::ref(notSupportedResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
}
@@ -3093,39 +3100,44 @@ TEST(TestAudioTrackGraph, PlatformProcessingNonNativeToNativeSwitch)
InSequence s;
// On first driver start.
EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(noneResult))))
graph, 0, Eq(std::ref(noneResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After param update.
EXPECT_CALL(*firstListener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(noiseResult))))
NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult(
graph, 1, Eq(std::ref(noiseResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After second param update.
EXPECT_CALL(*firstListener,
NotifySetRequestedInputProcessingParams(
graph, 2, CUBEB_INPUT_PROCESSING_PARAM_NONE));
EXPECT_CALL(*firstListener, NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NONE,
Eq(std::ref(noneResult))))
graph, 2, Eq(std::ref(noneResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// On non-native device's start.
EXPECT_CALL(*secondListener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(echoResult))))
NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION));
EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult(
graph, 1, Eq(std::ref(echoResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After switch to native device for second device.
EXPECT_CALL(*firstListener, Disconnect);
EXPECT_CALL(*secondListener, Disconnect);
EXPECT_CALL(*secondListener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION,
Eq(std::ref(echoResult))))
NotifySetRequestedInputProcessingParams(
graph, 1, CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION));
EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult(
graph, 1, Eq(std::ref(echoResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
// After param update.
EXPECT_CALL(*secondListener,
NotifySetRequestedInputProcessingParamsResult(
graph, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION,
Eq(std::ref(noiseResult))))
NotifySetRequestedInputProcessingParams(
graph, 2, CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION));
EXPECT_CALL(*secondListener, NotifySetRequestedInputProcessingParamsResult(
graph, 2, Eq(std::ref(noiseResult))))
.WillOnce([&] { ++numProcessingParamsResults; });
EXPECT_CALL(*secondListener, Disconnect);
}

7
dom/media/gtest/TestDeviceInputTrack.cpp
View File

@@ -136,8 +136,13 @@ TEST_F(TestDeviceInputTrack, DeviceInputConsumerTrack) {
};
void DeviceChanged(MediaTrackGraph* aGraph) override { /* Ignored */ }
void Disconnect(MediaTrackGraph* aGraph) override { /* Ignored */ };
void NotifySetRequestedInputProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams) override {
/* Ignored */
}
void NotifySetRequestedInputProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult) override {
/* Ignored */
}

33
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
View File

@@ -421,15 +421,42 @@ AudioInputProcessing::AudioInputProcessing(uint32_t aMaxChannelCount)
}
void AudioInputProcessing::Disconnect(MediaTrackGraph* aGraph) {
// This method is just for asserts.
aGraph->AssertOnGraphThread();
mPlatformProcessingSetGeneration = 0;
mPlatformProcessingSetParams = CUBEB_INPUT_PROCESSING_PARAM_NONE;
ApplySettingsInternal(aGraph, mSettings);
}
void AudioInputProcessing::NotifySetRequestedInputProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams) {
aGraph->AssertOnGraphThread();
MOZ_ASSERT(aGeneration >= mPlatformProcessingSetGeneration);
if (aGeneration <= mPlatformProcessingSetGeneration) {
return;
}
mPlatformProcessingSetGeneration = aGeneration;
cubeb_input_processing_params intersection =
mPlatformProcessingSetParams & aRequestedParams;
LOG("AudioInputProcessing %p platform processing params being applied are "
"now %s (Gen %d). Assuming %s while waiting for the result.",
this, CubebUtils::ProcessingParamsToString(aRequestedParams).get(),
aGeneration, CubebUtils::ProcessingParamsToString(intersection).get());
if (mPlatformProcessingSetParams == intersection) {
LOG("AudioInputProcessing %p intersection %s of platform processing params "
"already applied. Doing nothing.",
this, CubebUtils::ProcessingParamsToString(intersection).get());
return;
}
mPlatformProcessingSetParams = intersection;
ApplySettingsInternal(aGraph, mSettings);
}
void AudioInputProcessing::NotifySetRequestedInputProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult) {
aGraph->AssertOnGraphThread();
if (aRequestedParams != RequestedInputProcessingParams(aGraph)) {
if (aGeneration != mPlatformProcessingSetGeneration) {
// This is a result from an old request, wait for a more recent one.
return;
}

16
dom/media/webrtc/MediaEngineWebRTCAudio.h
View File

@@ -135,13 +135,20 @@ class AudioInputProcessing : public AudioDataListener {
void Disconnect(MediaTrackGraph* aGraph) override;
// Prepare for a change to platform processing params by assuming the platform
// applies the intersection of the already applied params and
// aRequestedParams, and set the software config accordingly.
void NotifySetRequestedInputProcessingParams(
MediaTrackGraph* aGraph, int aGeneration,
cubeb_input_processing_params aRequestedParams) override;
// Handle the result of an async operation to set processing params on a cubeb
// stream. If the operation succeeded, disable the applied processing params
// from the software processing config. If the operation failed, request
// platform processing to be disabled so as to not prevent a cubeb stream from
// being created.
void NotifySetRequestedInputProcessingParamsResult(
MediaTrackGraph* aGraph, cubeb_input_processing_params aRequestedParams,
MediaTrackGraph* aGraph, int aGeneration,
const Result<cubeb_input_processing_params, int>& aResult) override;
void PacketizeAndProcess(AudioProcessingTrack* aTrack,
@@ -209,12 +216,15 @@ class AudioInputProcessing : public AudioDataListener {
// When false, RequestedInputProcessingParams() returns no params, resulting
// in platform processing getting disabled in the platform.
bool mPlatformProcessingEnabled = false;
// The generation tracking the latest requested set of platform processing
// params.
int mPlatformProcessingSetGeneration = -1;
// The latest error notified to us through
// NotifySetRequestedInputProcessingParamsResult, or Nothing if the latest
// request was successful, or if a request is pending a result.
Maybe<int> mPlatformProcessingSetError;
// The processing params currently applied in the platform. This allows
// adapting the AudioProcessingConfig accordingly.
// The processing params currently applied, or about to be applied, in the
// platform. This allows adapting the AudioProcessingConfig accordingly.
cubeb_input_processing_params mPlatformProcessingSetParams =
CUBEB_INPUT_PROCESSING_PARAM_NONE;
// Buffer for up to one 10ms packet of planar mixed audio output for the