From 159900ec32cb59090260e6e863799ca1b61d83e3 Mon Sep 17 00:00:00 2001 From: Andrew Osmond Date: Wed, 6 Oct 2021 14:41:18 +0000 Subject: [PATCH] Bug 1732115 - Part 4. Add/use reorienting decoding pipeline. r=tnikkel This part hooks up the swizzling code from earlier to the AVIF and JPEG decoder pipelines. Differential Revision: https://phabricator.services.mozilla.com/D126382 --- image/Decoder.h | 2 +- image/SurfacePipe.cpp | 63 +++++++++++++++++++++++++- image/SurfacePipe.h | 43 ++++++++++++++++-- image/SurfacePipeFactory.h | 78 ++++++++++++++++++++++++++++++++ image/decoders/nsAVIFDecoder.cpp | 5 +- image/decoders/nsJPEGDecoder.cpp | 6 +-- 6 files changed, 183 insertions(+), 14 deletions(-) diff --git a/image/Decoder.h b/image/Decoder.h index f5b7d3907796..afb123688078 100644 --- a/image/Decoder.h +++ b/image/Decoder.h @@ -446,7 +446,7 @@ class Decoder { friend class DecoderTestHelper; friend class nsBMPDecoder; friend class nsICODecoder; - friend class PalettedSurfaceSink; + friend class ReorientSurfaceSink; friend class SurfaceSink; virtual ~Decoder(); diff --git a/image/SurfacePipe.cpp b/image/SurfacePipe.cpp index 29825c65fd6f..2e4a86daefd1 100644 --- a/image/SurfacePipe.cpp +++ b/image/SurfacePipe.cpp @@ -36,12 +36,12 @@ uint8_t* AbstractSurfaceSink::DoResetToFirstRow() { return GetRowPointer(); } -uint8_t* AbstractSurfaceSink::DoAdvanceRowFromBuffer(const uint8_t* aInputRow) { +uint8_t* SurfaceSink::DoAdvanceRowFromBuffer(const uint8_t* aInputRow) { CopyInputRow(aInputRow); return DoAdvanceRow(); } -uint8_t* AbstractSurfaceSink::DoAdvanceRow() { +uint8_t* SurfaceSink::DoAdvanceRow() { if (mRow >= uint32_t(InputSize().height)) { return nullptr; } @@ -99,5 +99,64 @@ uint8_t* SurfaceSink::GetRowPointer() const { return rowPtr; } +uint8_t* ReorientSurfaceSink::DoAdvanceRowFromBuffer(const uint8_t* aInputRow) { + if (mRow >= uint32_t(InputSize().height)) { + return nullptr; + } + + IntRect dirty = mReorientFn(aInputRow, mRow, mImageData, mSurfaceSize, + mSurfaceSize.width * sizeof(uint32_t)); + auto orientedDirty = OrientedIntRect::FromUnknownRect(dirty); + mInvalidRect.UnionRect(mInvalidRect, orientedDirty); + + mRow = min(uint32_t(InputSize().height), mRow + 1); + + return mRow < uint32_t(InputSize().height) ? GetRowPointer() : nullptr; +} + +uint8_t* ReorientSurfaceSink::DoAdvanceRow() { + return DoAdvanceRowFromBuffer(mBuffer.get()); +} + +nsresult ReorientSurfaceSink::Configure(const ReorientSurfaceConfig& aConfig) { + mSurfaceSize = aConfig.mOutputSize.ToUnknownSize(); + + // Allocate the frame. + // XXX(seth): Once every Decoder subclass uses SurfacePipe, we probably want + // to allocate the frame directly here and get rid of Decoder::AllocateFrame + // altogether. + nsresult rv = + aConfig.mDecoder->AllocateFrame(mSurfaceSize, aConfig.mFormat, Nothing()); + if (NS_FAILED(rv)) { + return rv; + } + + // The filters above us need the unoriented size as the input. + auto inputSize = + aConfig.mOrientation.ToUnoriented(aConfig.mOutputSize).ToUnknownSize(); + mBuffer.reset(new (fallible) uint8_t[inputSize.width * sizeof(uint32_t)]); + if (MOZ_UNLIKELY(!mBuffer)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + memset(mBuffer.get(), 0xFF, inputSize.width * sizeof(uint32_t)); + + mReorientFn = ReorientRow(aConfig.mOrientation); + MOZ_ASSERT(mReorientFn); + + mImageData = aConfig.mDecoder->mImageData; + mImageDataLength = aConfig.mDecoder->mImageDataLength; + + MOZ_ASSERT(mImageData); + MOZ_ASSERT(uint64_t(mImageDataLength) == uint64_t(mSurfaceSize.width) * + uint64_t(mSurfaceSize.height) * + sizeof(uint32_t)); + + ConfigureFilter(inputSize, sizeof(uint32_t)); + return NS_OK; +} + +uint8_t* ReorientSurfaceSink::GetRowPointer() const { return mBuffer.get(); } + } // namespace image } // namespace mozilla diff --git a/image/SurfacePipe.h b/image/SurfacePipe.h index f9bbed9255ba..0c3533c9f940 100644 --- a/image/SurfacePipe.h +++ b/image/SurfacePipe.h @@ -34,6 +34,7 @@ #include "mozilla/Unused.h" #include "mozilla/Variant.h" #include "mozilla/gfx/2D.h" +#include "mozilla/gfx/Swizzle.h" #include "nsDebug.h" #include "Orientation.h" @@ -758,7 +759,7 @@ class SurfacePipe { /** * AbstractSurfaceSink contains shared implementation for both SurfaceSink and - * PalettedSurfaceSink. + * ReorientSurfaceSink. */ class AbstractSurfaceSink : public SurfaceFilter { public: @@ -772,8 +773,6 @@ class AbstractSurfaceSink : public SurfaceFilter { protected: uint8_t* DoResetToFirstRow() final; - uint8_t* DoAdvanceRowFromBuffer(const uint8_t* aInputRow) final; - uint8_t* DoAdvanceRow() final; virtual uint8_t* GetRowPointer() const = 0; OrientedIntRect @@ -799,7 +798,7 @@ struct SurfaceConfig { /** * A sink for surfaces. It handles the allocation of the surface and protects - * against buffer overflow. This sink should be used for images. + * against buffer overflow. This sink should be used for most images. * * Sinks must always be at the end of the SurfaceFilter chain. */ @@ -808,7 +807,41 @@ class SurfaceSink final : public AbstractSurfaceSink { nsresult Configure(const SurfaceConfig& aConfig); protected: - uint8_t* GetRowPointer() const override; + uint8_t* DoAdvanceRowFromBuffer(const uint8_t* aInputRow) final; + uint8_t* DoAdvanceRow() final; + uint8_t* GetRowPointer() const final; +}; + +class ReorientSurfaceSink; + +/// A configuration struct for ReorientSurfaceSink. +struct ReorientSurfaceConfig { + using Filter = ReorientSurfaceSink; + Decoder* mDecoder; /// Which Decoder to use to allocate the surface. + OrientedIntSize mOutputSize; /// The size of the surface. + gfx::SurfaceFormat mFormat; /// The surface format (BGRA or BGRX). + Orientation mOrientation; /// The desired orientation of the surface data. +}; + +/** + * A sink for surfaces. It handles the allocation of the surface and protects + * against buffer overflow. This sink should be used for images which have a + * non-identity orientation which we want to apply during decoding. + * + * Sinks must always be at the end of the SurfaceFilter chain. + */ +class ReorientSurfaceSink final : public AbstractSurfaceSink { + public: + nsresult Configure(const ReorientSurfaceConfig& aConfig); + + protected: + uint8_t* DoAdvanceRowFromBuffer(const uint8_t* aInputRow) final; + uint8_t* DoAdvanceRow() final; + uint8_t* GetRowPointer() const final; + + UniquePtr mBuffer; + gfx::ReorientRowFn mReorientFn; + gfx::IntSize mSurfaceSize; }; } // namespace image diff --git a/image/SurfacePipeFactory.h b/image/SurfacePipeFactory.h index fa1e4360fd03..004ce349a68c 100644 --- a/image/SurfacePipeFactory.h +++ b/image/SurfacePipeFactory.h @@ -116,6 +116,8 @@ class SurfacePipeFactory { MOZ_ASSERT(aOutFormat == gfx::SurfaceFormat::OS_RGBA || aOutFormat == gfx::SurfaceFormat::OS_RGBX); + MOZ_ASSERT(aDecoder->GetOrientation().IsIdentity()); + const bool inFormatRgb = aInFormat == gfx::SurfaceFormat::R8G8B8; const bool inFormatOpaque = aInFormat == gfx::SurfaceFormat::OS_RGBX || @@ -580,6 +582,82 @@ class SurfacePipeFactory { return pipe; } + /** + * Creates and initializes a reorienting SurfacePipe. + * + * @param aDecoder The decoder whose current frame the SurfacePipe will write + * to. + * @param aInputSize The original size of the image. + * @param aOutputSize The size the SurfacePipe should output. Must be the same + * as @aInputSize or smaller. If smaller, the image will be + * downscaled during decoding. + * @param aFormat The surface format of the image; generally B8G8R8A8 or + * B8G8R8X8. + * @param aOrientation The orientation of the image. + * + * @return A SurfacePipe if the parameters allowed one to be created + * successfully, or Nothing() if the SurfacePipe could not be + * initialized. + */ + static Maybe CreateReorientSurfacePipe( + Decoder* aDecoder, const OrientedIntSize& aInputSize, + const OrientedIntSize& aOutputSize, gfx::SurfaceFormat aFormat, + qcms_transform* aTransform, const Orientation& aOrientation) { + const bool downscale = aInputSize != aOutputSize; + const bool colorManagement = aTransform != nullptr; + + // Construct configurations for the SurfaceFilters. Note that the order of + // these filters is significant. We want to deinterlace or interpolate raw + // input rows, before any other transformations, and we want to remove the + // frame rect (which may involve adding blank rows or columns to the image) + // before any downscaling, so that the new rows and columns are taken into + // account. + DownscalingConfig downscalingConfig{ + aOrientation.ToUnoriented(aInputSize).ToUnknownSize(), aFormat}; + ColorManagementConfig colorManagementConfig{aTransform}; + SurfaceConfig surfaceConfig{aDecoder, aOutputSize.ToUnknownSize(), aFormat, + /* mFlipVertically */ false, + /* mAnimParams */ Nothing()}; + ReorientSurfaceConfig reorientSurfaceConfig{aDecoder, aOutputSize, aFormat, + aOrientation}; + + Maybe pipe; + + if (aOrientation.IsIdentity()) { + if (colorManagement) { + if (downscale) { + pipe = + MakePipe(downscalingConfig, colorManagementConfig, surfaceConfig); + } else { // (downscale is false) + pipe = MakePipe(colorManagementConfig, surfaceConfig); + } + } else { // (colorManagement is false) + if (downscale) { + pipe = MakePipe(downscalingConfig, surfaceConfig); + } else { // (downscale is false) + pipe = MakePipe(surfaceConfig); + } + } + } else { // (orientation is not identity) + if (colorManagement) { + if (downscale) { + pipe = MakePipe(downscalingConfig, colorManagementConfig, + reorientSurfaceConfig); + } else { // (downscale is false) + pipe = MakePipe(colorManagementConfig, reorientSurfaceConfig); + } + } else { // (colorManagement is false) + if (downscale) { + pipe = MakePipe(downscalingConfig, reorientSurfaceConfig); + } else { // (downscale is false) + pipe = MakePipe(reorientSurfaceConfig); + } + } + } + + return pipe; + } + private: template static Maybe MakePipe(const Configs&... aConfigs) { diff --git a/image/decoders/nsAVIFDecoder.cpp b/image/decoders/nsAVIFDecoder.cpp index 7682439a9535..9d1b7496a129 100644 --- a/image/decoders/nsAVIFDecoder.cpp +++ b/image/decoders/nsAVIFDecoder.cpp @@ -1554,9 +1554,8 @@ nsAVIFDecoder::DecodeResult nsAVIFDecoder::Decode( MOZ_LOG(sAVIFLog, LogLevel::Debug, ("[this=%p] calling SurfacePipeFactory::CreateSurfacePipe", this)); - Maybe pipe = SurfacePipeFactory::CreateSurfacePipe( - this, Size(), OutputSize(), FullFrame(), format, format, Nothing(), - mTransform, SurfacePipeFlags()); + Maybe pipe = SurfacePipeFactory::CreateReorientSurfacePipe( + this, Size(), OutputSize(), format, mTransform, GetOrientation()); if (!pipe) { MOZ_LOG(sAVIFLog, LogLevel::Debug, diff --git a/image/decoders/nsJPEGDecoder.cpp b/image/decoders/nsJPEGDecoder.cpp index ab08ed03dac5..9ea440361df7 100644 --- a/image/decoders/nsJPEGDecoder.cpp +++ b/image/decoders/nsJPEGDecoder.cpp @@ -360,9 +360,9 @@ LexerTransition nsJPEGDecoder::ReadJPEGData( qcms_transform* pipeTransform = mInfo.out_color_space != JCS_GRAYSCALE ? mTransform : nullptr; - Maybe pipe = SurfacePipeFactory::CreateSurfacePipe( - this, Size(), OutputSize(), FullFrame(), SurfaceFormat::OS_RGBX, - SurfaceFormat::OS_RGBX, Nothing(), pipeTransform, SurfacePipeFlags()); + Maybe pipe = SurfacePipeFactory::CreateReorientSurfacePipe( + this, Size(), OutputSize(), SurfaceFormat::OS_RGBX, pipeTransform, + GetOrientation()); if (!pipe) { mState = JPEG_ERROR; MOZ_LOG(sJPEGDecoderAccountingLog, LogLevel::Debug,