blob: f360cefe960f005ebc512e549240df65394fc2c1 [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "flutter/shell/common/rasterizer.h"
#include "flutter/shell/common/persistent_cache.h"
#include <utility>
#include "flutter/fml/time/time_delta.h"
#include "flutter/fml/time/time_point.h"
#include "third_party/skia/include/core/SkEncodedImageFormat.h"
#include "third_party/skia/include/core/SkImageEncoder.h"
#include "third_party/skia/include/core/SkPictureRecorder.h"
#include "third_party/skia/include/core/SkSerialProcs.h"
#include "third_party/skia/include/core/SkSurface.h"
#include "third_party/skia/include/core/SkSurfaceCharacterization.h"
#include "third_party/skia/include/utils/SkBase64.h"
namespace flutter {
// The rasterizer will tell Skia to purge cached resources that have not been
// used within this interval.
static constexpr std::chrono::milliseconds kSkiaCleanupExpiration(15000);
// TODO(dnfield): Remove this once internal embedders have caught up.
static Rasterizer::DummyDelegate dummy_delegate_;
Rasterizer::Rasterizer(
TaskRunners task_runners,
std::unique_ptr<flutter::CompositorContext> compositor_context)
: Rasterizer(dummy_delegate_,
std::move(task_runners),
std::move(compositor_context)) {}
Rasterizer::Rasterizer(Delegate& delegate, TaskRunners task_runners)
: Rasterizer(delegate,
std::move(task_runners),
std::make_unique<flutter::CompositorContext>(
delegate.GetFrameBudget())) {}
Rasterizer::Rasterizer(
Delegate& delegate,
TaskRunners task_runners,
std::unique_ptr<flutter::CompositorContext> compositor_context)
: delegate_(delegate),
task_runners_(std::move(task_runners)),
compositor_context_(std::move(compositor_context)),
user_override_resource_cache_bytes_(false),
weak_factory_(this) {
FML_DCHECK(compositor_context_);
}
Rasterizer::~Rasterizer() = default;
fml::WeakPtr<Rasterizer> Rasterizer::GetWeakPtr() const {
return weak_factory_.GetWeakPtr();
}
fml::WeakPtr<SnapshotDelegate> Rasterizer::GetSnapshotDelegate() const {
return weak_factory_.GetWeakPtr();
}
void Rasterizer::Setup(std::unique_ptr<Surface> surface) {
surface_ = std::move(surface);
if (max_cache_bytes_.has_value()) {
SetResourceCacheMaxBytes(max_cache_bytes_.value(),
user_override_resource_cache_bytes_);
}
compositor_context_->OnGrContextCreated();
if (surface_->GetExternalViewEmbedder()) {
const auto platform_id =
task_runners_.GetPlatformTaskRunner()->GetTaskQueueId();
const auto gpu_id = task_runners_.GetRasterTaskRunner()->GetTaskQueueId();
raster_thread_merger_ =
fml::MakeRefCounted<fml::RasterThreadMerger>(platform_id, gpu_id);
}
}
void Rasterizer::Teardown() {
compositor_context_->OnGrContextDestroyed();
surface_.reset();
last_layer_tree_.reset();
}
void Rasterizer::NotifyLowMemoryWarning() const {
if (!surface_) {
FML_DLOG(INFO) << "Rasterizer::PurgeCaches called with no surface.";
return;
}
auto context = surface_->GetContext();
if (!context) {
FML_DLOG(INFO) << "Rasterizer::PurgeCaches called with no GrContext.";
return;
}
context->freeGpuResources();
}
flutter::TextureRegistry* Rasterizer::GetTextureRegistry() {
return &compositor_context_->texture_registry();
}
flutter::LayerTree* Rasterizer::GetLastLayerTree() {
return last_layer_tree_.get();
}
void Rasterizer::DrawLastLayerTree() {
if (!last_layer_tree_ || !surface_) {
return;
}
DrawToSurface(*last_layer_tree_);
}
void Rasterizer::Draw(fml::RefPtr<Pipeline<flutter::LayerTree>> pipeline) {
TRACE_EVENT0("flutter", "GPURasterizer::Draw");
if (raster_thread_merger_ &&
!raster_thread_merger_->IsOnRasterizingThread()) {
// we yield and let this frame be serviced on the right thread.
return;
}
FML_DCHECK(task_runners_.GetRasterTaskRunner()->RunsTasksOnCurrentThread());
RasterStatus raster_status = RasterStatus::kFailed;
Pipeline<flutter::LayerTree>::Consumer consumer =
[&](std::unique_ptr<LayerTree> layer_tree) {
raster_status = DoDraw(std::move(layer_tree));
};
PipelineConsumeResult consume_result = pipeline->Consume(consumer);
// if the raster status is to resubmit the frame, we push the frame to the
// front of the queue and also change the consume status to more available.
if (raster_status == RasterStatus::kResubmit) {
auto front_continuation = pipeline->ProduceIfEmpty();
front_continuation.Complete(std::move(resubmitted_layer_tree_));
} else if (raster_status == RasterStatus::kEnqueuePipeline) {
consume_result = PipelineConsumeResult::MoreAvailable;
}
// Consume as many pipeline items as possible. But yield the event loop
// between successive tries.
switch (consume_result) {
case PipelineConsumeResult::MoreAvailable: {
task_runners_.GetRasterTaskRunner()->PostTask(
[weak_this = weak_factory_.GetWeakPtr(), pipeline]() {
if (weak_this) {
weak_this->Draw(pipeline);
}
});
break;
}
default:
break;
}
}
sk_sp<SkImage> Rasterizer::DoMakeRasterSnapshot(
SkISize size,
std::function<void(SkCanvas*)> draw_callback) {
TRACE_EVENT0("flutter", __FUNCTION__);
sk_sp<SkSurface> surface;
SkImageInfo image_info = SkImageInfo::MakeN32Premul(
size.width(), size.height(), SkColorSpace::MakeSRGB());
if (surface_ == nullptr || surface_->GetContext() == nullptr) {
// Raster surface is fine if there is no on screen surface. This might
// happen in case of software rendering.
surface = SkSurface::MakeRaster(image_info);
} else {
if (!surface_->MakeRenderContextCurrent()) {
return nullptr;
}
// When there is an on screen surface, we need a render target SkSurface
// because we want to access texture backed images.
surface = SkSurface::MakeRenderTarget(surface_->GetContext(), // context
SkBudgeted::kNo, // budgeted
image_info // image info
);
}
if (surface == nullptr || surface->getCanvas() == nullptr) {
return nullptr;
}
draw_callback(surface->getCanvas());
surface->getCanvas()->flush();
sk_sp<SkImage> device_snapshot;
{
TRACE_EVENT0("flutter", "MakeDeviceSnpashot");
device_snapshot = surface->makeImageSnapshot();
}
if (device_snapshot == nullptr) {
return nullptr;
}
{
TRACE_EVENT0("flutter", "DeviceHostTransfer");
if (auto raster_image = device_snapshot->makeRasterImage()) {
return raster_image;
}
}
return nullptr;
}
sk_sp<SkImage> Rasterizer::MakeRasterSnapshot(sk_sp<SkPicture> picture,
SkISize picture_size) {
return DoMakeRasterSnapshot(picture_size,
[picture = std::move(picture)](SkCanvas* canvas) {
canvas->drawPicture(picture);
});
}
sk_sp<SkImage> Rasterizer::ConvertToRasterImage(sk_sp<SkImage> image) {
TRACE_EVENT0("flutter", __FUNCTION__);
// If the rasterizer does not have a surface with a GrContext, then it will
// be unable to render a cross-context SkImage. The caller will need to
// create the raster image on the IO thread.
if (surface_ == nullptr || surface_->GetContext() == nullptr) {
return nullptr;
}
if (image == nullptr) {
return nullptr;
}
return DoMakeRasterSnapshot(image->dimensions(),
[image = std::move(image)](SkCanvas* canvas) {
canvas->drawImage(image, 0, 0);
});
}
RasterStatus Rasterizer::DoDraw(
std::unique_ptr<flutter::LayerTree> layer_tree) {
FML_DCHECK(task_runners_.GetRasterTaskRunner()->RunsTasksOnCurrentThread());
if (!layer_tree || !surface_) {
return RasterStatus::kFailed;
}
FrameTiming timing;
const fml::TimePoint frame_target_time = layer_tree->target_time();
timing.Set(FrameTiming::kBuildStart, layer_tree->build_start());
timing.Set(FrameTiming::kBuildFinish, layer_tree->build_finish());
timing.Set(FrameTiming::kRasterStart, fml::TimePoint::Now());
PersistentCache* persistent_cache = PersistentCache::GetCacheForProcess();
persistent_cache->ResetStoredNewShaders();
RasterStatus raster_status = DrawToSurface(*layer_tree);
if (raster_status == RasterStatus::kSuccess) {
last_layer_tree_ = std::move(layer_tree);
} else if (raster_status == RasterStatus::kResubmit) {
resubmitted_layer_tree_ = std::move(layer_tree);
return raster_status;
}
if (persistent_cache->IsDumpingSkp() &&
persistent_cache->StoredNewShaders()) {
auto screenshot =
ScreenshotLastLayerTree(ScreenshotType::SkiaPicture, false);
persistent_cache->DumpSkp(*screenshot.data);
}
// TODO(liyuqian): in Fuchsia, the rasterization doesn't finish when
// Rasterizer::DoDraw finishes. Future work is needed to adapt the timestamp
// for Fuchsia to capture SceneUpdateContext::ExecutePaintTasks.
const auto raster_finish_time = fml::TimePoint::Now();
timing.Set(FrameTiming::kRasterFinish, raster_finish_time);
delegate_.OnFrameRasterized(timing);
if (raster_finish_time > frame_target_time) {
fml::TimePoint latest_frame_target_time =
delegate_.GetLatestFrameTargetTime();
const auto frame_budget_millis = delegate_.GetFrameBudget().count();
if (latest_frame_target_time < raster_finish_time) {
latest_frame_target_time =
latest_frame_target_time +
fml::TimeDelta::FromMillisecondsF(frame_budget_millis);
}
const auto frame_lag =
(latest_frame_target_time - frame_target_time).ToMillisecondsF();
const int vsync_transitions_missed = round(frame_lag / frame_budget_millis);
fml::tracing::TraceEventAsyncComplete(
"flutter", // category
"SceneDisplayLag", // name
frame_target_time, // begin_time
raster_finish_time, // end_time
"frame_target_time", // arg_key_1
frame_target_time, // arg_val_1
"current_frame_target_time", // arg_key_2
latest_frame_target_time, // arg_val_2
"vsync_transitions_missed", // arg_key_3
vsync_transitions_missed // arg_val_3
);
}
// Pipeline pressure is applied from a couple of places:
// rasterizer: When there are more items as of the time of Consume.
// animator (via shell): Frame gets produces every vsync.
// Enqueing here is to account for the following scenario:
// T = 1
// - one item (A) in the pipeline
// - rasterizer starts (and merges the threads)
// - pipeline consume result says no items to process
// T = 2
// - animator produces (B) to the pipeline
// - applies pipeline pressure via platform thread.
// T = 3
// - rasterizes finished (and un-merges the threads)
// - |Draw| for B yields as its on the wrong thread.
// This enqueue ensures that we attempt to consume from the right
// thread one more time after un-merge.
if (raster_thread_merger_) {
if (raster_thread_merger_->DecrementLease() ==
fml::RasterThreadStatus::kUnmergedNow) {
return RasterStatus::kEnqueuePipeline;
}
}
return raster_status;
}
RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
TRACE_EVENT0("flutter", "Rasterizer::DrawToSurface");
FML_DCHECK(surface_);
auto frame = surface_->AcquireFrame(layer_tree.frame_size());
if (frame == nullptr) {
return RasterStatus::kFailed;
}
// There is no way for the compositor to know how long the layer tree
// construction took. Fortunately, the layer tree does. Grab that time
// for instrumentation.
compositor_context_->ui_time().SetLapTime(layer_tree.build_time());
auto* external_view_embedder = surface_->GetExternalViewEmbedder();
SkCanvas* embedder_root_canvas = nullptr;
if (external_view_embedder != nullptr) {
external_view_embedder->BeginFrame(layer_tree.frame_size(),
surface_->GetContext(),
layer_tree.device_pixel_ratio());
embedder_root_canvas = external_view_embedder->GetRootCanvas();
}
// If the external view embedder has specified an optional root surface, the
// root surface transformation is set by the embedder instead of
// having to apply it here.
SkMatrix root_surface_transformation =
embedder_root_canvas ? SkMatrix{} : surface_->GetRootTransformation();
auto root_surface_canvas =
embedder_root_canvas ? embedder_root_canvas : frame->SkiaCanvas();
auto compositor_frame = compositor_context_->AcquireFrame(
surface_->GetContext(), // skia GrContext
root_surface_canvas, // root surface canvas
external_view_embedder, // external view embedder
root_surface_transformation, // root surface transformation
true, // instrumentation enabled
frame->supports_readback(), // surface supports pixel reads
raster_thread_merger_ // thread merger
);
if (compositor_frame) {
RasterStatus raster_status = compositor_frame->Raster(layer_tree, false);
if (raster_status == RasterStatus::kFailed) {
return raster_status;
}
if (external_view_embedder != nullptr) {
external_view_embedder->SubmitFrame(surface_->GetContext(),
root_surface_canvas);
// The external view embedder may mutate the root surface canvas while
// submitting the frame.
// Therefore, submit the final frame after asking the external view
// embedder to submit the frame.
frame->Submit();
external_view_embedder->FinishFrame();
} else {
frame->Submit();
}
FireNextFrameCallbackIfPresent();
if (surface_->GetContext()) {
TRACE_EVENT0("flutter", "PerformDeferredSkiaCleanup");
surface_->GetContext()->performDeferredCleanup(kSkiaCleanupExpiration);
}
return raster_status;
}
return RasterStatus::kFailed;
}
static sk_sp<SkData> SerializeTypeface(SkTypeface* typeface, void* ctx) {
return typeface->serialize(SkTypeface::SerializeBehavior::kDoIncludeData);
}
static sk_sp<SkData> ScreenshotLayerTreeAsPicture(
flutter::LayerTree* tree,
flutter::CompositorContext& compositor_context) {
FML_DCHECK(tree != nullptr);
SkPictureRecorder recorder;
recorder.beginRecording(
SkRect::MakeWH(tree->frame_size().width(), tree->frame_size().height()));
SkMatrix root_surface_transformation;
root_surface_transformation.reset();
// TODO(amirh): figure out how to take a screenshot with embedded UIView.
// https://github.com/flutter/flutter/issues/23435
auto frame = compositor_context.AcquireFrame(
nullptr, recorder.getRecordingCanvas(), nullptr,
root_surface_transformation, false, true, nullptr);
frame->Raster(*tree, true);
SkSerialProcs procs = {0};
procs.fTypefaceProc = SerializeTypeface;
return recorder.finishRecordingAsPicture()->serialize(&procs);
}
static sk_sp<SkSurface> CreateSnapshotSurface(GrContext* surface_context,
const SkISize& size) {
const auto image_info = SkImageInfo::MakeN32Premul(
size.width(), size.height(), SkColorSpace::MakeSRGB());
if (surface_context) {
// There is a rendering surface that may contain textures that are going to
// be referenced in the layer tree about to be drawn.
return SkSurface::MakeRenderTarget(surface_context, //
SkBudgeted::kNo, //
image_info //
);
}
// There is no rendering surface, assume no GPU textures are present and
// create a raster surface.
return SkSurface::MakeRaster(image_info);
}
static sk_sp<SkData> ScreenshotLayerTreeAsImage(
flutter::LayerTree* tree,
flutter::CompositorContext& compositor_context,
GrContext* surface_context,
bool compressed) {
// Attempt to create a snapshot surface depending on whether we have access to
// a valid GPU rendering context.
auto snapshot_surface =
CreateSnapshotSurface(surface_context, tree->frame_size());
if (snapshot_surface == nullptr) {
FML_LOG(ERROR) << "Screenshot: unable to create snapshot surface";
return nullptr;
}
// Draw the current layer tree into the snapshot surface.
auto* canvas = snapshot_surface->getCanvas();
// There is no root surface transformation for the screenshot layer. Reset the
// matrix to identity.
SkMatrix root_surface_transformation;
root_surface_transformation.reset();
// We want to ensure we call the base method for
// CompositorContext::AcquireFrame instead of the platform-specific method.
// Specifically, Fuchsia's CompositorContext handles the rendering surface
// itself which means that we will still continue to render to the onscreen
// surface if we don't call the base method.
auto frame = compositor_context.flutter::CompositorContext::AcquireFrame(
surface_context, canvas, nullptr, root_surface_transformation, false,
true, nullptr);
canvas->clear(SK_ColorTRANSPARENT);
frame->Raster(*tree, true);
canvas->flush();
// Prepare an image from the surface, this image may potentially be on th GPU.
auto potentially_gpu_snapshot = snapshot_surface->makeImageSnapshot();
if (!potentially_gpu_snapshot) {
FML_LOG(ERROR) << "Screenshot: unable to make image screenshot";
return nullptr;
}
// Copy the GPU image snapshot into CPU memory.
auto cpu_snapshot = potentially_gpu_snapshot->makeRasterImage();
if (!cpu_snapshot) {
FML_LOG(ERROR) << "Screenshot: unable to make raster image";
return nullptr;
}
// If the caller want the pixels to be compressed, there is a Skia utility to
// compress to PNG. Use that.
if (compressed) {
return cpu_snapshot->encodeToData();
}
// Copy it into a bitmap and return the same.
SkPixmap pixmap;
if (!cpu_snapshot->peekPixels(&pixmap)) {
FML_LOG(ERROR) << "Screenshot: unable to obtain bitmap pixels";
return nullptr;
}
return SkData::MakeWithCopy(pixmap.addr32(), pixmap.computeByteSize());
}
Rasterizer::Screenshot Rasterizer::ScreenshotLastLayerTree(
Rasterizer::ScreenshotType type,
bool base64_encode) {
auto* layer_tree = GetLastLayerTree();
if (layer_tree == nullptr) {
FML_LOG(ERROR) << "Last layer tree was null when screenshotting.";
return {};
}
sk_sp<SkData> data = nullptr;
GrContext* surface_context = surface_ ? surface_->GetContext() : nullptr;
switch (type) {
case ScreenshotType::SkiaPicture:
data = ScreenshotLayerTreeAsPicture(layer_tree, *compositor_context_);
break;
case ScreenshotType::UncompressedImage:
data = ScreenshotLayerTreeAsImage(layer_tree, *compositor_context_,
surface_context, false);
break;
case ScreenshotType::CompressedImage:
data = ScreenshotLayerTreeAsImage(layer_tree, *compositor_context_,
surface_context, true);
break;
}
if (data == nullptr) {
FML_LOG(ERROR) << "Screenshot data was null.";
return {};
}
if (base64_encode) {
size_t b64_size = SkBase64::Encode(data->data(), data->size(), nullptr);
auto b64_data = SkData::MakeUninitialized(b64_size);
SkBase64::Encode(data->data(), data->size(), b64_data->writable_data());
return Rasterizer::Screenshot{b64_data, layer_tree->frame_size()};
}
return Rasterizer::Screenshot{data, layer_tree->frame_size()};
}
void Rasterizer::SetNextFrameCallback(const fml::closure& callback) {
next_frame_callback_ = callback;
}
void Rasterizer::FireNextFrameCallbackIfPresent() {
if (!next_frame_callback_) {
return;
}
// It is safe for the callback to set a new callback.
auto callback = next_frame_callback_;
next_frame_callback_ = nullptr;
callback();
}
void Rasterizer::SetResourceCacheMaxBytes(size_t max_bytes, bool from_user) {
user_override_resource_cache_bytes_ |= from_user;
if (!from_user && user_override_resource_cache_bytes_) {
// We should not update the setting here if a user has explicitly set a
// value for this over the flutter/skia channel.
return;
}
max_cache_bytes_ = max_bytes;
if (!surface_) {
return;
}
GrContext* context = surface_->GetContext();
if (context) {
int max_resources;
context->getResourceCacheLimits(&max_resources, nullptr);
context->setResourceCacheLimits(max_resources, max_bytes);
}
}
std::optional<size_t> Rasterizer::GetResourceCacheMaxBytes() const {
if (!surface_) {
return std::nullopt;
}
GrContext* context = surface_->GetContext();
if (context) {
size_t max_bytes;
context->getResourceCacheLimits(nullptr, &max_bytes);
return max_bytes;
}
return std::nullopt;
}
Rasterizer::Screenshot::Screenshot() {}
Rasterizer::Screenshot::Screenshot(sk_sp<SkData> p_data, SkISize p_size)
: data(std::move(p_data)), frame_size(p_size) {}
Rasterizer::Screenshot::Screenshot(const Screenshot& other) = default;
Rasterizer::Screenshot::~Screenshot() = default;
} // namespace flutter