| // Copyright 2013 The Flutter Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "flutter/lib/ui/painting/image_decoder.h" |
| |
| #include <algorithm> |
| |
| #include "flutter/fml/make_copyable.h" |
| #include "third_party/skia/include/codec/SkCodec.h" |
| #include "third_party/skia/src/codec/SkCodecImageGenerator.h" |
| |
| namespace flutter { |
| namespace { |
| |
| constexpr double kAspectRatioChangedThreshold = 0.01; |
| |
| } // namespace |
| |
| ImageDecoder::ImageDecoder( |
| TaskRunners runners, |
| std::shared_ptr<fml::ConcurrentTaskRunner> concurrent_task_runner, |
| fml::WeakPtr<IOManager> io_manager) |
| : runners_(std::move(runners)), |
| concurrent_task_runner_(std::move(concurrent_task_runner)), |
| io_manager_(std::move(io_manager)), |
| weak_factory_(this) { |
| FML_DCHECK(runners_.IsValid()); |
| FML_DCHECK(runners_.GetUITaskRunner()->RunsTasksOnCurrentThread()) |
| << "The image decoder must be created & collected on the UI thread."; |
| } |
| |
| ImageDecoder::~ImageDecoder() = default; |
| |
| static double AspectRatio(const SkISize& size) { |
| return static_cast<double>(size.width()) / size.height(); |
| } |
| |
| // Get the updated dimensions of the image. If both dimensions are specified, |
| // use them. If one of them is specified, respect the one that is and use the |
| // aspect ratio to calculate the other. If neither dimension is specified, use |
| // intrinsic dimensions of the image. |
| static SkISize GetResizedDimensions(SkISize current_size, |
| std::optional<uint32_t> target_width, |
| std::optional<uint32_t> target_height, |
| ImageUpscalingMode image_upscaling) { |
| if (current_size.isEmpty()) { |
| return SkISize::MakeEmpty(); |
| } |
| |
| if (image_upscaling == ImageUpscalingMode::kNotAllowed) { |
| if (target_width) { |
| target_width = std::min(current_size.width(), |
| static_cast<int32_t>(target_width.value())); |
| } |
| if (target_height) { |
| target_height = std::min(current_size.height(), |
| static_cast<int32_t>(target_height.value())); |
| } |
| } |
| |
| if (target_width && target_height) { |
| return SkISize::Make(target_width.value(), target_height.value()); |
| } |
| |
| const auto aspect_ratio = AspectRatio(current_size); |
| |
| if (target_width) { |
| return SkISize::Make(target_width.value(), |
| target_width.value() / aspect_ratio); |
| } |
| |
| if (target_height) { |
| return SkISize::Make(target_height.value() * aspect_ratio, |
| target_height.value()); |
| } |
| |
| return current_size; |
| } |
| |
| static sk_sp<SkImage> ResizeRasterImage(sk_sp<SkImage> image, |
| const SkISize& resized_dimensions, |
| const fml::tracing::TraceFlow& flow) { |
| FML_DCHECK(!image->isTextureBacked()); |
| |
| TRACE_EVENT0("flutter", __FUNCTION__); |
| flow.Step(__FUNCTION__); |
| |
| if (resized_dimensions.isEmpty()) { |
| FML_LOG(ERROR) << "Could not resize to empty dimensions."; |
| return nullptr; |
| } |
| |
| if (image->dimensions() == resized_dimensions) { |
| return image->makeRasterImage(); |
| } |
| |
| // TODO(dnfield): remove this in favor of clearer constraints. |
| // https://github.com/flutter/flutter/issues/59578 |
| const bool aspect_ratio_changed = |
| std::abs(AspectRatio(resized_dimensions) - |
| AspectRatio(image->dimensions())) > kAspectRatioChangedThreshold; |
| if (aspect_ratio_changed) { |
| // This is probably a bug. If a user passes dimensions that change the |
| // aspect ratio in a "caching" context that's probably not working as |
| // intended and rather a signal that the API is hard to use. |
| FML_LOG(WARNING) |
| << "Aspect ratio changes. Are cache(Height|Width) used correctly?"; |
| } |
| |
| const auto scaled_image_info = |
| image->imageInfo().makeDimensions(resized_dimensions); |
| |
| SkBitmap scaled_bitmap; |
| if (!scaled_bitmap.tryAllocPixels(scaled_image_info)) { |
| FML_LOG(ERROR) << "Failed to allocate memory for bitmap of size " |
| << scaled_image_info.computeMinByteSize() << "B"; |
| return nullptr; |
| } |
| |
| if (!image->scalePixels(scaled_bitmap.pixmap(), kLow_SkFilterQuality, |
| SkImage::kDisallow_CachingHint)) { |
| FML_LOG(ERROR) << "Could not scale pixels"; |
| return nullptr; |
| } |
| |
| // Marking this as immutable makes the MakeFromBitmap call share the pixels |
| // instead of copying. |
| scaled_bitmap.setImmutable(); |
| |
| auto scaled_image = SkImage::MakeFromBitmap(scaled_bitmap); |
| if (!scaled_image) { |
| FML_LOG(ERROR) << "Could not create a scaled image from a scaled bitmap."; |
| return nullptr; |
| } |
| |
| return scaled_image; |
| } |
| |
| static sk_sp<SkImage> ImageFromDecompressedData( |
| sk_sp<SkData> data, |
| ImageDecoder::ImageInfo info, |
| std::optional<uint32_t> target_width, |
| std::optional<uint32_t> target_height, |
| ImageUpscalingMode image_upscaling, |
| const fml::tracing::TraceFlow& flow) { |
| TRACE_EVENT0("flutter", __FUNCTION__); |
| flow.Step(__FUNCTION__); |
| auto image = SkImage::MakeRasterData(info.sk_info, data, info.row_bytes); |
| |
| if (!image) { |
| FML_LOG(ERROR) << "Could not create image from decompressed bytes."; |
| return nullptr; |
| } |
| |
| if (!target_width && !target_height) { |
| // No resizing requested. Just rasterize the image. |
| return image->makeRasterImage(); |
| } |
| |
| auto resized_dimensions = GetResizedDimensions( |
| image->dimensions(), target_width, target_height, image_upscaling); |
| |
| return ResizeRasterImage(std::move(image), resized_dimensions, flow); |
| } |
| |
| sk_sp<SkImage> ImageFromCompressedData(sk_sp<SkData> data, |
| std::optional<uint32_t> target_width, |
| std::optional<uint32_t> target_height, |
| ImageUpscalingMode image_upscaling, |
| const fml::tracing::TraceFlow& flow) { |
| TRACE_EVENT0("flutter", __FUNCTION__); |
| flow.Step(__FUNCTION__); |
| |
| if (!target_width && !target_height) { |
| // No resizing requested. Just decode & rasterize the image. |
| return SkImage::MakeFromEncoded(data)->makeRasterImage(); |
| } |
| |
| auto codec = SkCodec::MakeFromData(data); |
| if (codec == nullptr) { |
| return nullptr; |
| } |
| |
| const auto* codec_ptr = codec.get(); |
| |
| // Note that we cannot read the dimensions from the codec since they don't |
| // respect image orientation provided e.g. in EXIF data. |
| auto image_generator = SkCodecImageGenerator::MakeFromCodec(std::move(codec)); |
| const auto& source_dimensions = image_generator->getInfo().dimensions(); |
| |
| auto resized_dimensions = GetResizedDimensions( |
| source_dimensions, target_width, target_height, image_upscaling); |
| |
| // No resize needed. |
| if (resized_dimensions == source_dimensions) { |
| return SkImage::MakeFromEncoded(data)->makeRasterImage(); |
| } |
| |
| auto decode_dimensions = codec_ptr->getScaledDimensions( |
| std::max(static_cast<double>(resized_dimensions.width()) / |
| source_dimensions.width(), |
| static_cast<double>(resized_dimensions.height()) / |
| source_dimensions.height())); |
| |
| // If the codec supports efficient sub-pixel decoding, decoded at a resolution |
| // close to the target resolution before resizing. |
| if (decode_dimensions != codec_ptr->dimensions()) { |
| if (source_dimensions != codec_ptr->dimensions()) { |
| decode_dimensions = |
| SkISize::Make(decode_dimensions.height(), decode_dimensions.width()); |
| } |
| |
| auto scaled_image_info = |
| image_generator->getInfo().makeDimensions(decode_dimensions); |
| |
| SkBitmap scaled_bitmap; |
| if (!scaled_bitmap.tryAllocPixels(scaled_image_info)) { |
| FML_LOG(ERROR) << "Failed to allocate memory for bitmap of size " |
| << scaled_image_info.computeMinByteSize() << "B"; |
| return nullptr; |
| } |
| |
| const auto& pixmap = scaled_bitmap.pixmap(); |
| if (image_generator->getPixels(pixmap.info(), pixmap.writable_addr(), |
| pixmap.rowBytes())) { |
| // Marking this as immutable makes the MakeFromBitmap call share |
| // the pixels instead of copying. |
| scaled_bitmap.setImmutable(); |
| |
| auto decoded_image = SkImage::MakeFromBitmap(scaled_bitmap); |
| FML_DCHECK(decoded_image); |
| if (!decoded_image) { |
| FML_LOG(ERROR) |
| << "Could not create a scaled image from a scaled bitmap."; |
| return nullptr; |
| } |
| return ResizeRasterImage(std::move(decoded_image), resized_dimensions, |
| flow); |
| } |
| } |
| |
| auto image = SkImage::MakeFromEncoded(data); |
| if (!image) { |
| return nullptr; |
| } |
| |
| return ResizeRasterImage(std::move(image), resized_dimensions, flow); |
| } |
| |
| static SkiaGPUObject<SkImage> UploadRasterImage( |
| sk_sp<SkImage> image, |
| fml::WeakPtr<IOManager> io_manager, |
| const fml::tracing::TraceFlow& flow) { |
| TRACE_EVENT0("flutter", __FUNCTION__); |
| flow.Step(__FUNCTION__); |
| |
| // Should not already be a texture image because that is the entire point of |
| // the this method. |
| FML_DCHECK(!image->isTextureBacked()); |
| |
| if (!io_manager->GetResourceContext() || !io_manager->GetSkiaUnrefQueue()) { |
| FML_LOG(ERROR) |
| << "Could not acquire context of release queue for texture upload."; |
| return {}; |
| } |
| |
| SkPixmap pixmap; |
| if (!image->peekPixels(&pixmap)) { |
| FML_LOG(ERROR) << "Could not peek pixels of image for texture upload."; |
| return {}; |
| } |
| |
| SkiaGPUObject<SkImage> result; |
| io_manager->GetIsGpuDisabledSyncSwitch()->Execute( |
| fml::SyncSwitch::Handlers() |
| .SetIfTrue([&result, &pixmap, &image] { |
| SkSafeRef(image.get()); |
| sk_sp<SkImage> texture_image = SkImage::MakeFromRaster( |
| pixmap, |
| [](const void* pixels, SkImage::ReleaseContext context) { |
| SkSafeUnref(static_cast<SkImage*>(context)); |
| }, |
| image.get()); |
| result = {std::move(texture_image), nullptr}; |
| }) |
| .SetIfFalse([&result, context = io_manager->GetResourceContext(), |
| &pixmap, queue = io_manager->GetSkiaUnrefQueue()] { |
| TRACE_EVENT0("flutter", "MakeCrossContextImageFromPixmap"); |
| sk_sp<SkImage> texture_image = SkImage::MakeCrossContextFromPixmap( |
| context.get(), // context |
| pixmap, // pixmap |
| true, // buildMips, |
| true // limitToMaxTextureSize |
| ); |
| if (!texture_image) { |
| FML_LOG(ERROR) << "Could not make x-context image."; |
| result = {}; |
| } else { |
| result = {std::move(texture_image), queue}; |
| } |
| })); |
| |
| return result; |
| } |
| |
| void ImageDecoder::Decode(ImageDescriptor descriptor, |
| const ImageResult& callback) { |
| TRACE_EVENT0("flutter", __FUNCTION__); |
| fml::tracing::TraceFlow flow(__FUNCTION__); |
| |
| FML_DCHECK(callback); |
| FML_DCHECK(runners_.GetUITaskRunner()->RunsTasksOnCurrentThread()); |
| |
| // Always service the callback on the UI thread. |
| auto result = [callback, ui_runner = runners_.GetUITaskRunner()]( |
| SkiaGPUObject<SkImage> image, |
| fml::tracing::TraceFlow flow) { |
| ui_runner->PostTask(fml::MakeCopyable( |
| [callback, image = std::move(image), flow = std::move(flow)]() mutable { |
| // We are going to terminate the trace flow here. Flows cannot |
| // terminate without a base trace. Add one explicitly. |
| TRACE_EVENT0("flutter", "ImageDecodeCallback"); |
| flow.End(); |
| callback(std::move(image)); |
| })); |
| }; |
| |
| if (!descriptor.data || descriptor.data->size() == 0) { |
| result({}, std::move(flow)); |
| return; |
| } |
| |
| concurrent_task_runner_->PostTask( |
| fml::MakeCopyable([descriptor, // |
| io_manager = io_manager_, // |
| io_runner = runners_.GetIOTaskRunner(), // |
| result, // |
| flow = std::move(flow) // |
| ]() mutable { |
| // Step 1: Decompress the image. |
| // On Worker. |
| |
| auto decompressed = |
| descriptor.decompressed_image_info |
| ? ImageFromDecompressedData( |
| std::move(descriptor.data), // |
| descriptor.decompressed_image_info.value(), // |
| descriptor.target_width, // |
| descriptor.target_height, // |
| descriptor.image_upscaling, // |
| flow // |
| ) |
| : ImageFromCompressedData(std::move(descriptor.data), // |
| descriptor.target_width, // |
| descriptor.target_height, // |
| descriptor.image_upscaling, // |
| flow); |
| |
| if (!decompressed) { |
| FML_LOG(ERROR) << "Could not decompress image."; |
| result({}, std::move(flow)); |
| return; |
| } |
| |
| // Step 2: Update the image to the GPU. |
| // On IO Thread. |
| |
| io_runner->PostTask(fml::MakeCopyable([io_manager, decompressed, result, |
| flow = |
| std::move(flow)]() mutable { |
| if (!io_manager) { |
| FML_LOG(ERROR) << "Could not acquire IO manager."; |
| return result({}, std::move(flow)); |
| } |
| |
| // If the IO manager does not have a resource context, the caller |
| // might not have set one or a software backend could be in use. |
| // Either way, just return the image as-is. |
| if (!io_manager->GetResourceContext()) { |
| result({std::move(decompressed), io_manager->GetSkiaUnrefQueue()}, |
| std::move(flow)); |
| return; |
| } |
| |
| auto uploaded = |
| UploadRasterImage(std::move(decompressed), io_manager, flow); |
| |
| if (!uploaded.get()) { |
| FML_LOG(ERROR) << "Could not upload image to the GPU."; |
| result({}, std::move(flow)); |
| return; |
| } |
| |
| // Finally, all done. |
| result(std::move(uploaded), std::move(flow)); |
| })); |
| })); |
| } |
| |
| fml::WeakPtr<ImageDecoder> ImageDecoder::GetWeakPtr() const { |
| return weak_factory_.GetWeakPtr(); |
| } |
| |
| } // namespace flutter |