diff --git a/dom/media/mp4/MP4Demuxer.cpp b/dom/media/mp4/MP4Demuxer.cpp index 569b7896c691..4e82485f73bd 100644 --- a/dom/media/mp4/MP4Demuxer.cpp +++ b/dom/media/mp4/MP4Demuxer.cpp @@ -414,10 +414,11 @@ already_AddRefed MP4TrackDemuxer::GetNextSample() { if (mType == kH264 && !sample->mCrypto.IsEncrypted()) { H264::FrameType type = H264::GetFrameType(sample); switch (type) { - case H264::FrameType::I_FRAME: - [[fallthrough]]; + case H264::FrameType::I_FRAME_IDR: + case H264::FrameType::I_FRAME_OTHER: case H264::FrameType::OTHER: { - bool keyframe = type == H264::FrameType::I_FRAME; + bool keyframe = type == H264::FrameType::I_FRAME_OTHER || + type == H264::FrameType::I_FRAME_IDR; if (sample->mKeyframe != keyframe) { NS_WARNING(nsPrintfCString("Frame incorrectly marked as %skeyframe " "@ pts:%" PRId64 " dur:%" PRId64 diff --git a/dom/media/platforms/agnostic/bytestreams/H264.cpp b/dom/media/platforms/agnostic/bytestreams/H264.cpp index 226931463a49..e3c60bb35da8 100644 --- a/dom/media/platforms/agnostic/bytestreams/H264.cpp +++ b/dom/media/platforms/agnostic/bytestreams/H264.cpp @@ -958,18 +958,18 @@ uint32_t H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData) { int8_t nalType = AssertedCast(*p & 0x1f); if (nalType == H264_NAL_IDR_SLICE) { // IDR NAL. - return FrameType::I_FRAME; + return FrameType::I_FRAME_IDR; } if (nalType == H264_NAL_SEI) { RefPtr decodedNAL = DecodeNALUnit(p, nalLen); SEIRecoveryData data; if (DecodeRecoverySEI(decodedNAL, data)) { - return FrameType::I_FRAME; + return FrameType::I_FRAME_OTHER; } } else if (nalType == H264_NAL_SLICE) { RefPtr decodedNAL = DecodeNALUnit(p, nalLen); if (DecodeISlice(decodedNAL)) { - return FrameType::I_FRAME; + return FrameType::I_FRAME_OTHER; } } } diff --git a/dom/media/platforms/agnostic/bytestreams/H264.h b/dom/media/platforms/agnostic/bytestreams/H264.h index c62d6b15e135..bcf01f0555cc 100644 --- a/dom/media/platforms/agnostic/bytestreams/H264.h +++ b/dom/media/platforms/agnostic/bytestreams/H264.h @@ -516,7 +516,14 @@ class H264 { const mozilla::MediaByteBuffer* aExtraData); enum class FrameType { - I_FRAME, + // IDR is a special iframe, according to the spec T-REC-H.264-202408, 3.69 : + // "An IDR picture causes the decoding process to mark all reference + // pictures as "unused for reference" immediately after the decoding of the + // IDR picture. All coded pictures that follow an IDR picture in decoding + // order can be decoded without inter prediction from any picture that + // precedes the IDR picture in decoding order." + I_FRAME_IDR, + I_FRAME_OTHER, OTHER, INVALID, }; diff --git a/dom/media/platforms/wrappers/MediaChangeMonitor.cpp b/dom/media/platforms/wrappers/MediaChangeMonitor.cpp index 1840a99d9aed..46fd2be056be 100644 --- a/dom/media/platforms/wrappers/MediaChangeMonitor.cpp +++ b/dom/media/platforms/wrappers/MediaChangeMonitor.cpp @@ -74,10 +74,17 @@ static bool IsBeingProfiledOrLogEnabled() { class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { public: - explicit H264ChangeMonitor(const VideoInfo& aInfo, bool aFullParsing) - : mCurrentConfig(aInfo), mFullParsing(aFullParsing) { + explicit H264ChangeMonitor(const CreateDecoderParams& aParams) + : mCurrentConfig(aParams.VideoConfig()), + mFullParsing(aParams.mOptions.contains( + CreateDecoderParams::Option::FullH264Parsing)) +#ifdef MOZ_WMF_MEDIA_ENGINE + , + mIsMediaEnginePlayback(aParams.mMediaEngineId.isSome()) +#endif + { if (CanBeInstantiated()) { - UpdateConfigFromExtraData(aInfo.mExtraData); + UpdateConfigFromExtraData(mCurrentConfig.mExtraData); auto avcc = AVCCConfig::Parse(mCurrentConfig.mExtraData); if (avcc.isOk() && avcc.unwrap().NALUSize() != 4) { // `CheckForChange()` will use `AnnexB::ConvertSampleToAVCC()` to change @@ -169,14 +176,21 @@ class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { aSample->mTrackInfo = mTrackInfo; bool appendExtradata = aNeedKeyFrame; - if (aSample->mCrypto.IsEncrypted() && !mReceivedFirstEncryptedSample) { - LOG("Detected first encrypted sample [%" PRId64 ",%" PRId64 - "], keyframe=%d", - aSample->mTime.ToMicroseconds(), - aSample->GetEndTime().ToMicroseconds(), aSample->mKeyframe); - mReceivedFirstEncryptedSample = true; - appendExtradata = true; +#ifdef MOZ_WMF_MEDIA_ENGINE + // The error SPR_E_INVALID_H264_SLICE_HEADERS is caused by the media engine + // being unable to handle an IDR frame without a valid SPS. Therefore, we + // ensure that SPS should always be presented in the bytestream for all IDR + // frames. + if (mIsMediaEnginePlayback && + H264::GetFrameType(aSample) == H264::FrameType::I_FRAME_IDR) { + RefPtr extradata = H264::ExtractExtraData(aSample); + appendExtradata = aNeedKeyFrame || !H264::HasSPS(extradata); + LOG("%s need to append extradata for IDR sample [%" PRId64 ",%" PRId64 + "]", + appendExtradata ? "Do" : "No", aSample->mTime.ToMicroseconds(), + aSample->GetEndTime().ToMicroseconds()); } +#endif if (aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB) { auto res = AnnexB::ConvertAVCCSampleToAnnexB(aSample, appendExtradata); @@ -189,8 +203,6 @@ class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { return NS_OK; } - void Flush() override { mReceivedFirstEncryptedSample = false; } - private: void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData) { SPSData spsdata; @@ -224,14 +236,13 @@ class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { VideoInfo mCurrentConfig; uint32_t mStreamID = 0; const bool mFullParsing; +#ifdef MOZ_WMF_MEDIA_ENGINE + // True if the playback is performed by Windows Media Foundation Engine. + const bool mIsMediaEnginePlayback; +#endif bool mGotSPS = false; RefPtr mTrackInfo; RefPtr mPreviousExtraData; - - // This ensures the first encrypted sample always includes all necessary - // information for decoding, as some decoders, such as MediaEngine, require - // SPS/PPS to be appended during the clearlead-to-encrypted transition. - bool mReceivedFirstEncryptedSample = false; }; class HEVCChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor { @@ -820,9 +831,7 @@ RefPtr MediaChangeMonitor::Create( changeMonitor = MakeUnique(config); } else { MOZ_ASSERT(MP4Decoder::IsH264(config.mMimeType)); - changeMonitor = MakeUnique( - config, aParams.mOptions.contains( - CreateDecoderParams::Option::FullH264Parsing)); + changeMonitor = MakeUnique(aParams); } } else { MOZ_ASSERT(MP4Decoder::IsAAC(aParams.AudioConfig().mMimeType));