From c73fcf79f77a96a4ede5f4a4d3e50a1b5d833219 Mon Sep 17 00:00:00 2001
From: Qiu Jianlin <[email protected]>
Date: Thu, 16 Apr 2026 15:50:25 -0700
Subject: [PATCH] Fix GPU UAF via early return in D3D12 VEA.

Early return for encoding failures, and avoid reset GPU command
allocator when the command list is still being executed.

Bug: 500174874
Change-Id: Idbdc1cc355fae3a0fed1b7673321497eeb6272dd
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/7751041
Reviewed-by: Eugene Zemtsov <[email protected]>
Commit-Queue: Qiu, Jianlin <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1616167}
---

diff --git a/media/gpu/windows/d3d12_video_encode_accelerator.cc b/media/gpu/windows/d3d12_video_encode_accelerator.cc
index 76e1a0a..c629c2a 100644
--- a/media/gpu/windows/d3d12_video_encode_accelerator.cc
+++ b/media/gpu/windows/d3d12_video_encode_accelerator.cc
@@ -845,6 +845,10 @@
 void D3D12VideoEncodeAccelerator::TryEncodeFrames() {
   DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
 
+  if (error_occurred_) {
+    return;
+  }
+
   while (!input_frames_queue_.empty() && !bitstream_buffers_.empty()) {
     auto& next_input = input_frames_queue_.front();
     if (next_input.resolving_shared_image ||
@@ -856,9 +860,12 @@
       break;
     }
 
-    DoEncodeTask(next_input, bitstream_buffers_.front());
+    const bool success = DoEncodeTask(next_input, bitstream_buffers_.front());
     input_frames_queue_.pop_front();
     bitstream_buffers_.pop();
+    if (!success) {
+      break;
+    }
   }
 
   if (flush_requested_ && input_frames_queue_.empty()) {
@@ -869,7 +876,7 @@
   }
 }
 
-void D3D12VideoEncodeAccelerator::DoEncodeTask(
+bool D3D12VideoEncodeAccelerator::DoEncodeTask(
     const InputFrameRef& input_frame,
     const BitstreamBuffer& bitstream_buffer) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
@@ -882,9 +889,10 @@
     } else {
       frame = ConvertToMemoryMappedFrame(std::move(frame));
       if (!frame) {
-        return NotifyError(
+        NotifyError(
             {EncoderStatus::Codes::kInvalidInputFrame,
              "Failed to convert shared memory mappable SI for encoding"});
+        return false;
       }
       picture_buffer = CreateResourceForSharedMemoryVideoFrame(*frame);
     }
@@ -893,19 +901,22 @@
   } else if (frame->HasSharedImage()) {
     picture_buffer = input_frame.resolved_picture;
   } else {
-    return NotifyError({EncoderStatus::Codes::kInvalidInputFrame,
-                        "Unsupported frame storage type for encoding"});
+    NotifyError({EncoderStatus::Codes::kInvalidInputFrame,
+                 "Unsupported frame storage type for encoding"});
+    return false;
   }
   if (!picture_buffer.resource) {
-    return NotifyError({EncoderStatus::Codes::kInvalidInputFrame,
-                        "Failed to create input_texture"});
+    NotifyError({EncoderStatus::Codes::kInvalidInputFrame,
+                 "Failed to create input_texture"});
+    return false;
   }
 
   auto result_or_error =
       encoder_->Encode(picture_buffer, frame->ColorSpace(), bitstream_buffer,
                        input_frame.options);
   if (!result_or_error.has_value()) {
-    return NotifyError(std::move(result_or_error).error());
+    NotifyError(std::move(result_or_error).error());
+    return false;
   }
 
   D3D12VideoEncodeDelegate::EncodeResult result =
@@ -924,6 +935,7 @@
   child_task_runner_->PostTask(
       FROM_HERE, BindOnce(&Client::BitstreamBufferReady, client_,
                           result.bitstream_buffer_id, result.metadata));
+  return true;
 }
 
 void D3D12VideoEncodeAccelerator::DestroyTask() {
@@ -933,26 +945,36 @@
 }
 
 void D3D12VideoEncodeAccelerator::NotifyError(EncoderStatus status) {
+  // We return here when `error_occurred_` was already true, as this is not the
+  // first error that is reported.
+  if (error_occurred_.exchange(true)) {
+    return;
+  }
+
+  CHECK(!status.is_ok());
   base::UmaHistogramEnumeration(
       GetEncoderStatusHistogramName(config_.output_profile), status.code());
 
   if (!child_task_runner_->RunsTasksInCurrentSequence()) {
     child_task_runner_->PostTask(
-        FROM_HERE, BindOnce(&D3D12VideoEncodeAccelerator::NotifyError,
-                            child_weak_this_, std::move(status)));
+        FROM_HERE,
+        BindOnce(&D3D12VideoEncodeAccelerator::NotifyErrorOnChildSequence,
+                 child_weak_this_, std::move(status)));
     return;
   }
 
-  CHECK(!status.is_ok());
+  NotifyErrorOnChildSequence(std::move(status));
+}
+
+void D3D12VideoEncodeAccelerator::NotifyErrorOnChildSequence(
+    EncoderStatus status) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(child_sequence_checker_);
   MEDIA_LOG(ERROR, media_log_)
       << "D3D12VEA error " << static_cast<int32_t>(status.code()) << ": "
       << status.message();
-  if (!error_occurred_) {
-    if (client_) {
-      client_->NotifyErrorStatus(status);
-      client_ptr_factory_->InvalidateWeakPtrs();
-    }
-    error_occurred_ = true;
+  if (client_) {
+    client_->NotifyErrorStatus(status);
+    client_ptr_factory_->InvalidateWeakPtrs();
   }
 }
 
diff --git a/media/gpu/windows/d3d12_video_encode_accelerator.h b/media/gpu/windows/d3d12_video_encode_accelerator.h
index e58f120a..770b5d4 100644
--- a/media/gpu/windows/d3d12_video_encode_accelerator.h
+++ b/media/gpu/windows/d3d12_video_encode_accelerator.h
@@ -11,6 +11,7 @@
 
 #include <wrl.h>
 
+#include <atomic>
 #include <vector>
 
 #include "base/containers/circular_deque.h"
@@ -124,7 +125,8 @@
   void EncodeTask(scoped_refptr<VideoFrame> frame,
                   const VideoEncoder::EncodeOptions& options);
 
-  void DoEncodeTask(const InputFrameRef& input_frame,
+  // Returns false if an error was encountered.
+  bool DoEncodeTask(const InputFrameRef& input_frame,
                     const BitstreamBuffer& bitstream_buffer);
 
   void TryEncodeFrames();
@@ -139,6 +141,8 @@
 
   void NotifyError(EncoderStatus status);
 
+  void NotifyErrorOnChildSequence(EncoderStatus status);
+
   // Invoked when the CommandBufferHelper is available.
   void OnCommandBufferHelperAvailable(GetCommandBufferHelperResult result);
 
@@ -183,7 +187,8 @@
   base::WeakPtr<Client> client_;
   std::unique_ptr<MediaLog> media_log_;
 
-  bool error_occurred_ = false;
+  // Whether an encoding error has occurred.
+  std::atomic<bool> error_occurred_ = false;
 
   // True if Destroy() has been called.
   bool destroy_requested_ GUARDED_BY_CONTEXT(child_sequence_checker_) = false;
diff --git a/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc b/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
index de844fce..be1176f2 100644
--- a/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
+++ b/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
@@ -485,4 +485,107 @@
   Mock::VerifyAndClearExpectations(&client_);
 }
 
+// Verifies that when DoEncodeTask fails, TryEncodeFrames breaks out of the
+// loop and does not process subsequent frames.
+TEST_F(D3D12VideoEncodeAcceleratorTest, EncodeErrorStopsProcessingNextFrames) {

diff --git a/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc b/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
index de844fce..be1176f2 100644
--- a/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
+++ b/media/gpu/windows/d3d12_video_encode_accelerator_unittest.cc
@@ -485,4 +485,107 @@
   Mock::VerifyAndClearExpectations(&client_);
 }
 
+// Verifies that when DoEncodeTask fails, TryEncodeFrames breaks out of the
+// loop and does not process subsequent frames.
+TEST_F(D3D12VideoEncodeAcceleratorTest, EncodeErrorStopsProcessingNextFrames) {
+  auto* d3d12_video_encode_accelerator =
+      static_cast<D3D12VideoEncodeAccelerator*>(
+          video_encode_accelerator_.get());
+
+  // Set up a factory that makes Encode fail on the first call.
+  class FailFirstEncodeFactory : public D3D12VideoEncodeAccelerator::
+                                     VideoEncodeDelegateFactoryInterface {
+   public:
+    int encode_call_count_ = 0;
+
+    std::unique_ptr<D3D12VideoEncodeDelegate> CreateVideoEncodeDelegate(
+        ID3D12VideoDevice3* video_device,
+        VideoCodecProfile profile) override {
+      gpu::GpuDriverBugWorkarounds gpu_workarounds{};
+      auto encoder_delegate =
+          std::make_unique<NiceMock<MockVideoEncoderDelegate>>(
+              video_device, gpu_workarounds, profile);
+      ON_CALL(*encoder_delegate, Initialize(_))
+          .WillByDefault(Return(EncoderStatus::Codes::kOk));
+      ON_CALL(*encoder_delegate, GetMaxNumOfRefFrames())
+          .WillByDefault(Return(16));
+      ON_CALL(*encoder_delegate, GetMaxNumOfManualRefBuffers())
+          .WillByDefault(Return(0));
+      ON_CALL(*encoder_delegate, Encode(_, _, _, _))
+          .WillByDefault(
+              [this](D3D12PictureBuffer, const gfx::ColorSpace&,
+                     const BitstreamBuffer& bitstream_buffer,
+                     const VideoEncoder::EncodeOptions&)
+                  -> EncoderStatus::Or<D3D12VideoEncodeDelegate::EncodeResult> {
+                ++encode_call_count_;
+                if (encode_call_count_ == 1) {
+                  return EncoderStatus(
+                      EncoderStatus::Codes::kBadReferenceBuffer,
+                      "Simulated encode failure");
+                }
+                return D3D12VideoEncodeDelegate::EncodeResult{
+                    bitstream_buffer.id()};
+              });
+      return std::move(encoder_delegate);
+    }
+
+    VideoEncodeAccelerator::SupportedProfiles GetSupportedProfiles(
+        ID3D12VideoDevice3* video_device,
+        const std::vector<D3D12_VIDEO_ENCODER_CODEC>& codecs) override {
+      VideoEncodeAccelerator::SupportedProfile profile(kSupportedProfile,
+                                                       kSupportedSize, 30, 1);
+      profile.scalability_modes.push_back(SVCScalabilityMode::kL1T1);
+      profile.gpu_supported_pixel_formats.push_back(PIXEL_FORMAT_NV12);
+      profile.gpu_supported_pixel_formats.push_back(PIXEL_FORMAT_BGRA);
+      profile.supports_gpu_shared_images = true;
+      return {profile};
+    }
+  };
+
+  auto fail_factory = std::make_unique<FailFirstEncodeFactory>();
+  auto* fail_factory_ptr = fail_factory.get();
+  d3d12_video_encode_accelerator->SetEncoderFactoryForTesting(
+      std::move(fail_factory));
+
+  auto supported_profiles =
+      d3d12_video_encode_accelerator->GetSupportedProfiles();
+  ASSERT_FALSE(supported_profiles.empty());
+  auto profile = supported_profiles.front();
+  auto config = SupportedProfileToConfig(profile);
+
+  unsigned bitstream_buffer_count = 0;
+  size_t bitstream_buffer_size = 0;
+  EXPECT_CALL(*client_, RequireBitstreamBuffers(_, _, _))
+      .WillOnce(
+          [&](unsigned int count, const gfx::Size& size, size_t size_in_bytes) {
+            bitstream_buffer_count = count;
+            bitstream_buffer_size = size_in_bytes;
+          });
+  EXPECT_TRUE(d3d12_video_encode_accelerator
+                  ->Initialize(config, client_.get(), media_log_->Clone())
+                  .is_ok());
+  WaitForEncoderTasksToComplete();
+  Mock::VerifyAndClearExpectations(client_.get());
+
+  // Queue two frames and two bitstream buffers so TryEncodeFrames has two
+  // items to process in its loop.
+  for (unsigned i = 0; i < 2; ++i) {
+    BitstreamBuffer bitstream_buffer(
+        i, base::UnsafeSharedMemoryRegion::Create(bitstream_buffer_size),
+        bitstream_buffer_size);
+    d3d12_video_encode_accelerator->UseOutputBitstreamBuffer(
+        std::move(bitstream_buffer));
+  }
+  for (unsigned i = 0; i < 2; ++i) {
+    d3d12_video_encode_accelerator->Encode(CreateTestVideoFrame(), false);
+  }
+
+  EXPECT_CALL(*client_, NotifyErrorStatus(_)).Times(1);
+  WaitForEncoderTasksToComplete();
+  // Encode should have been called only once: the first call fails, and the
+  // loop should break without attempting to encode the second frame.
+  EXPECT_EQ(fail_factory_ptr->encode_call_count_, 1);
+  Mock::VerifyAndClearExpectations(client_.get());
+}
+
 }  // namespace media
diff --git a/media/gpu/windows/d3d12_video_encode_delegate_unittest.cc b/media/gpu/windows/d3d12_video_encode_delegate_unittest.cc
index 2a57aeb..b67eb474 100644
--- a/media/gpu/windows/d3d12_video_encode_delegate_unittest.cc
+++ b/media/gpu/windows/d3d12_video_encode_delegate_unittest.cc
@@ -333,4 +333,52 @@
   EXPECT_EQ(result_or_error.code(), EncoderStatus::Codes::kBadReferenceBuffer);
 }
 
+TEST_F(D3D12VideoEncodeDelegateTest,
+       EncodeWithOutOfRangeReferenceBufferIndexFails) {
+  VideoEncodeAccelerator::Config config = GetDefaultH264Config();
+  ASSERT_TRUE(encoder_delegate_->Initialize(config).is_ok());
+
+  gfx::Size input_size = config.input_visible_size;
+  auto input_frame = CreateResource(input_size, config.input_format);
+  gfx::ColorSpace color_space = gfx::ColorSpace::CreateREC709();
+  constexpr size_t kPayloadSize = 1024;
+  auto shared_memory = base::UnsafeSharedMemoryRegion::Create(kPayloadSize);
+  BitstreamBuffer bitstream_buffer(0, shared_memory.Duplicate(), kPayloadSize);
+
+  VideoEncoder::EncodeOptions options;
+  // Use a single reference buffer with an index >= GetMaxNumOfManualRefBuffers.
+  options.reference_buffers.push_back(
+      static_cast<uint8_t>(encoder_delegate_->GetMaxNumOfManualRefBuffers()));
+
+  auto result_or_error = encoder_delegate_->Encode(input_frame, color_space,
+                                                   bitstream_buffer, options);
+
+  EXPECT_FALSE(result_or_error.has_value());
+  EXPECT_EQ(result_or_error.code(), EncoderStatus::Codes::kBadReferenceBuffer);
+}
+
+TEST_F(D3D12VideoEncodeDelegateTest,
+       EncodeWithOutOfRangeUpdateBufferIndexFails) {
+  VideoEncodeAccelerator::Config config = GetDefaultH264Config();
+  ASSERT_TRUE(encoder_delegate_->Initialize(config).is_ok());
+
+  gfx::Size input_size = config.input_visible_size;
+  auto input_frame = CreateResource(input_size, config.input_format);
+  gfx::ColorSpace color_space = gfx::ColorSpace::CreateREC709();
+  constexpr size_t kPayloadSize = 1024;
+  auto shared_memory = base::UnsafeSharedMemoryRegion::Create(kPayloadSize);
+  BitstreamBuffer bitstream_buffer(0, shared_memory.Duplicate(), kPayloadSize);
+
+  VideoEncoder::EncodeOptions options;
+  // Set update_buffer to a value >= GetMaxNumOfRefFrames.
+  options.update_buffer =
+      static_cast<uint8_t>(encoder_delegate_->GetMaxNumOfRefFrames());
+
+  auto result_or_error = encoder_delegate_->Encode(input_frame, color_space,
+                                                   bitstream_buffer, options);
+
+  EXPECT_FALSE(result_or_error.has_value());
+  EXPECT_EQ(result_or_error.code(), EncoderStatus::Codes::kBadReferenceBuffer);
+}
+
 }  // namespace media