CVE-2026-5890
Overview
Changed Functions
| Function | Change | Notes |
|---|---|---|
iftest/encode_api_test.cc |
modified | |
switchtest/encode_api_test.cc |
modified | |
fortest/encode_api_test.cc |
modified | |
TESTtest/encode_api_test.cc |
modified | |
ifvp9/encoder/vp9_pickmode.c |
modified |
Files Changed
test/acm_random.htest/encode_api_test.ccvp9/encoder/vp9_pickmode.c
Patch
From ab5ec7a1852f634b81d29ade3c5fa74056498973 Mon Sep 17 00:00:00 2001 From: James Zern <[email protected]> Date: Mon, 02 Mar 2026 14:51:36 -0800 Subject: [PATCH] vp9_pick_inter_mode: fix buf offsets w/scaled refs When calculating SAD for `NEWMV` with the `LAST_FRAME`, check whether a scaled reference frame is available and update `xd->plane[].pre[0]` offsets accordingly. This matches the behavior in `combined_motion_search()`. This fixes a heap overflow (read) when calculating the SAD when using scaled references. Bug: 487259772 Change-Id: I13cb02faa13d95b4f9dc4c8a49363c55d5e90efa --- diff --git a/test/acm_random.h b/test/acm_random.h index 6ebb600..7500cb8 100644 --- a/test/acm_random.h +++ b/test/acm_random.h @@ -52,6 +52,13 @@ return (value >> 19) & 0xfff; } + uint16_t Rand10() { + const uint32_t value = + random_.Generate(testing::internal::Random::kMaxRange); + // There's a bit more entropy in the upper bits of this implementation. + return (value >> 21) & 0x3ff; + } + uint8_t Rand8() { const uint32_t value = random_.Generate(testing::internal::Random::kMaxRange); diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc index ad323aa..f97dec1f 100644 --- a/test/encode_api_test.cc +++ b/test/encode_api_test.cc @@ -56,7 +56,8 @@ } vpx_image_t *CreateImage(vpx_bit_depth_t bit_depth, vpx_img_fmt_t fmt, - unsigned int width, unsigned int height) { + unsigned int width, unsigned int height, + libvpx_test::ACMRandom *rng = nullptr) { assert(fmt != VPX_IMG_FMT_NV12); if (bit_depth > VPX_BITS_8) { fmt = static_cast<vpx_img_fmt_t>(fmt | VPX_IMG_FMT_HIGHBITDEPTH); @@ -64,26 +65,38 @@ vpx_image_t *image = vpx_img_alloc(nullptr, fmt, width, height, 1); if (!image) return image; - const int val = 1 << (bit_depth - 1); + auto get_val = [bit_depth, &rng]() { + if (rng != nullptr) { + switch (bit_depth) { + case VPX_BITS_8: return static_cast<int>(rng->Rand8()); + case VPX_BITS_10: return static_cast<int>(rng->Rand10()); + case VPX_BITS_12: return static_cast<int>(rng->Rand12()); + } + } + return 1 << (bit_depth - 1); + }; + const int val_y = get_val(); + const int val_u = get_val(); + const int val_v = get_val(); const unsigned int uv_h = (image->d_h + image->y_chroma_shift) >> image->y_chroma_shift; const unsigned int uv_w = (image->d_w + image->x_chroma_shift) >> image->x_chroma_shift; if (bit_depth > VPX_BITS_8) { for (unsigned int i = 0; i < image->d_h; ++i) { - Memset16(image->planes[0] + i * image->stride[0], val, image->d_w); + Memset16(image->planes[0] + i * image->stride[0], val_y, image->d_w); } for (unsigned int i = 0; i < uv_h; ++i) { - Memset16(image->planes[1] + i * image->stride[1], val, uv_w); - Memset16(image->planes[2] + i * image->stride[2], val, uv_w); + Memset16(image->planes[1] + i * image->stride[1], val_u, uv_w); + Memset16(image->planes[2] + i * image->stride[2], val_v, uv_w); } } else { for (unsigned int i = 0; i < image->d_h; ++i) { - memset(image->planes[0] + i * image->stride[0], val, image->d_w); + memset(image->planes[0] + i * image->stride[0], val_y, image->d_w); } for (unsigned int i = 0; i < uv_h; ++i) { - memset(image->planes[1] + i * image->stride[1], val, uv_w); - memset(image->planes[2] + i * image->stride[2], val, uv_w); + memset(image->planes[1] + i * image->stride[1], val_u, uv_w); + memset(image->planes[2] + i * image->stride[2], val_v, uv_w); } } @@ -1249,7 +1262,9 @@ void Configure(unsigned int threads, unsigned int width, unsigned int height, vpx_rc_mode end_usage, vpx_enc_deadline_t deadline); - void Encode(bool key_frame); + // If `rng` is non-null, use it to generate the image values for encoding. + // Otherwise the midpoint of the configured bitdepth is used. + void Encode(bool key_frame, libvpx_test::ACMRandom *rng = nullptr); private: const int speed_; @@ -1316,10 +1331,10 @@ << vpx_codec_error_detail(&enc_); } -void VP9Encoder::Encode(bool key_frame) { +void VP9Encoder::Encode(bool key_frame, libvpx_test::ACMRandom *rng) { assert(initialized_); const vpx_codec_cx_pkt_t *pkt; - vpx_image_t *image = CreateImage(bit_depth_, fmt_, cfg_.g_w, cfg_.g_h); + vpx_image_t *image = CreateImage(bit_depth_, fmt_, cfg_.g_w, cfg_.g_h, rng); ASSERT_NE(image, nullptr); const vpx_enc_frame_flags_t frame_flags = key_frame ? VPX_EFLAG_FORCE_KF : 0; ASSERT_EQ( @@ -2388,6 +2403,25 @@ vpx_img_free(image); ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); } + +TEST(EncodeAPI, Buganizer487259772ScaledRefs) { + libvpx_test::ACMRandom rng; + VP9Encoder encoder(/*speed=*/7, /*row_mt=*/0, VPX_BITS_8, VPX_IMG_FMT_I420); + encoder.Configure(/*threads=*/1, /*width=*/478, /*height=*/755, VPX_VBR, + VPX_DL_REALTIME); + encoder.Configure(/*threads=*/1, /*width=*/387, /*height=*/438, VPX_VBR, + VPX_DL_REALTIME); + encoder.Encode(/*key_frame=*/false, &rng); + encoder.Encode(/*key_frame=*/true, &rng); + encoder.Encode(/*key_frame=*/false, &rng); + encoder.Encode(/*key_frame=*/false, &rng); + + encoder.Configure(/*threads=*/1, /*width=*/341, /*height=*/655, VPX_VBR, + VPX_DL_REALTIME); + encoder.Encode(/*key_frame=*/false, &rng); + encoder.Encode(/*key_frame=*/false, &rng); +} + #endif // CONFIG_VP9_ENCODER } // namespace diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c index b841385..7403d0a 100644 --- a/vp9/encoder/vp9_pickmode.c +++ b/vp9/encoder/vp9_pickmode.c @@ -2283,6 +2283,18 @@ // need to compute best_pred_sad which is only used to skip golden NEWMV. if (use_golden_nonzeromv && this_mode == NEWMV && ref_frame == LAST_FRAME && frame_mv[NEWMV][LAST_FRAME].as_int != INVALID_MV) { + struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0 } }; + const YV12_BUFFER_CONFIG *scaled_ref_frame = + vp9_get_scaled_ref_frame(cpi, ref_frame); + if (scaled_ref_frame) { + assert(scaled_ref_frame->y_width == cpi->Source->y_width && + scaled_ref_frame->y_height == cpi->Source->y_height); + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) backup_yv12[i] = xd->plane[i].pre[0]; + vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL); + } const int pre_stride = xd->plane[0].pre[0].stride; const uint8_t *const pre_buf = xd->plane[0].pre[0].buf + @@ -2291,6 +2303,9 @@ best_pred_sad = cpi->fn_ptr[bsize].sdf( x->plane[0].src.buf, x->plane[0].src.stride, pre_buf, pre_stride); x->pred_mv_sad[LAST_FRAME] = best_pred_sad; + if (scaled_ref_frame) { + for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i]; + } } if (this_mode != NEARESTMV && !comp_pred &&
Regression Test / PoC
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index ad323aa..f97dec1f 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -56,7 +56,8 @@
}
vpx_image_t *CreateImage(vpx_bit_depth_t bit_depth, vpx_img_fmt_t fmt,
- unsigned int width, unsigned int height) {
+ unsigned int width, unsigned int height,
+ libvpx_test::ACMRandom *rng = nullptr) {
assert(fmt != VPX_IMG_FMT_NV12);
if (bit_depth > VPX_BITS_8) {
fmt = static_cast<vpx_img_fmt_t>(fmt | VPX_IMG_FMT_HIGHBITDEPTH);
@@ -64,26 +65,38 @@
vpx_image_t *image = vpx_img_alloc(nullptr, fmt, width, height, 1);
if (!image) return image;
- const int val = 1 << (bit_depth - 1);
+ auto get_val = [bit_depth, &rng]() {
+ if (rng != nullptr) {
+ switch (bit_depth) {
+ case VPX_BITS_8: return static_cast<int>(rng->Rand8());
+ case VPX_BITS_10: return static_cast<int>(rng->Rand10());
+ case VPX_BITS_12: return static_cast<int>(rng->Rand12());
+ }
+ }
+ return 1 << (bit_depth - 1);
+ };
+ const int val_y = get_val();
+ const int val_u = get_val();
+ const int val_v = get_val();
const unsigned int uv_h =
(image->d_h + image->y_chroma_shift) >> image->y_chroma_shift;
const unsigned int uv_w =
(image->d_w + image->x_chroma_shift) >> image->x_chroma_shift;
if (bit_depth > VPX_BITS_8) {
for (unsigned int i = 0; i < image->d_h; ++i) {
- Memset16(image->planes[0] + i * image->stride[0], val, image->d_w);
+ Memset16(image->planes[0] + i * image->stride[0], val_y, image->d_w);
}
for (unsigned int i = 0; i < uv_h; ++i) {
- Memset16(image->planes[1] + i * image->stride[1], val, uv_w);
- Memset16(image->planes[2] + i * image->stride[2], val, uv_w);
+ Memset16(image->planes[1] + i * image->stride[1], val_u, uv_w);
+ Memset16(image->planes[2] + i * image->stride[2], val_v, uv_w);
}
} else {
for (unsigned int i = 0; i < image->d_h; ++i) {
- memset(image->planes[0] + i * image->stride[0], val, image->d_w);
+ memset(image->planes[0] + i * image->stride[0], val_y, image->d_w);
}
for (unsigned int i = 0; i < uv_h; ++i) {
- memset(image->planes[1] + i * image->stride[1], val, uv_w);
- memset(image->planes[2] + i * image->stride[2], val, uv_w);
+ memset(image->planes[1] + i * image->stride[1], val_u, uv_w);
+ memset(image->planes[2] + i * image->stride[2], val_v, uv_w);
}
}
@@ -1249,7 +1262,9 @@
void Configure(unsigned int threads, unsigned int width, unsigned int height,
vpx_rc_mode end_usage, vpx_enc_deadline_t deadline);
- void Encode(bool key_frame);
+ // If `rng` is non-null, use it to generate the image values for encoding.
+ // Otherwise the midpoint of the configured bitdepth is used.
+ void Encode(bool key_frame, libvpx_test::ACMRandom *rng = nullptr);
private:
const int speed_;
@@ -1316,10 +1331,10 @@
<< vpx_codec_error_detail(&enc_);
}
-void VP9Encoder::Encode(bool key_frame) {
+void VP9Encoder::Encode(bool key_frame, libvpx_test::ACMRandom *rng) {
assert(initialized_);
const vpx_codec_cx_pkt_t *pkt;
- vpx_image_t *image = CreateImage(bit_depth_, fmt_, cfg_.g_w, cfg_.g_h);
+ vpx_image_t *image = CreateImage(bit_depth_, fmt_, cfg_.g_w, cfg_.g_h, rng);
ASSERT_NE(image, nullptr);
const vpx_enc_frame_flags_t frame_flags = key_frame ? VPX_EFLAG_FORCE_KF : 0;
ASSERT_EQ(
@@ -2388,6 +2403,25 @@
vpx_img_free(image);
ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK);
}
+
+TEST(EncodeAPI, Buganizer487259772ScaledRefs) {
+ libvpx_test::ACMRandom rng;
+ VP9Encoder encoder(/*speed=*/7, /*row_mt=*/0, VPX_BITS_8, VPX_IMG_FMT_I420);
+ encoder.Configure(/*threads=*/1, /*width=*/478, /*height=*/755, VPX_VBR,
+ VPX_DL_REALTIME);
+ encoder.Configure(/*threads=*/1, /*width=*/387, /*height=*/438, VPX_VBR,
+ VPX_DL_REALTIME);
+ encoder.Encode(/*key_frame=*/false, &rng);
+ encoder.Encode(/*key_frame=*/true, &rng);
+ encoder.Encode(/*key_frame=*/false, &rng);
+ encoder.Encode(/*key_frame=*/false, &rng);
+
+ encoder.Configure(/*threads=*/1, /*width=*/341, /*height=*/655, VPX_VBR,
+ VPX_DL_REALTIME);
+ encoder.Encode(/*key_frame=*/false, &rng);
+ encoder.Encode(/*key_frame=*/false, &rng);
+}
+
#endif // CONFIG_VP9_ENCODER
} // namespace
Original Bug Report
Out-of-bounds heap read in VP9 encoder via WebCodecs VideoEncoder dimension reconfiguration
Report description
Out-of-bounds heap read in VP9 encoder via WebCodecs VideoEncoder dimension reconfiguration
Bug location
Where do you want to report your vulnerability?
Chrome VRP – Report security issues affecting the Chrome browser. See program rules
Which URL (or repository) have you found the vulnerability in?
https://chromium.googlesource.com/chromium/src/+/main/third_party/libvpx/source/libvpx/vp9/encoder/
The problem
Please describe the technical details of the vulnerability
Calling configure() with changing dimensions on a WebCodecs VideoEncoder (VP9 or AV1), followed by encode() without waiting for the previous encode to complete, causes memory corruption in both the VP9 and AV1 encode pipelines. The crash is stochastic and typically fires within 30 to 80 rounds of rapid dimension changes.
Tested on Windows 10 x64, reproduced on both ASAN Chrome 147.0.7696.0 and latest stable Chrome 145.0.7632.110.
The core issue is in Chrome’s OffloadingVideoEncoder, which dispatches Encode and ChangeOptions (the internal path for JS configure()) onto the encoder’s task queue without properly serializing them. When configure() arrives while codec-internal row-MT worker threads are still mid-encode, the reconfigure tears down internal state (reference frame buffers, context structures) while those threads still hold live pointers into the old allocations.
The ASAN task traces for both crashes confirm this interleaving:
OffloadingVideoEncoder::Encode
OffloadingVideoEncoder::WrapCallback
OffloadingVideoEncoder::ChangeOptions <- reconfigure while encode in flight
OffloadingVideoEncoder::WrapCallback
When the encoder gets reconfigured to different dimensions, internal reference frame buffers from earlier configurations remain in the encoder’s buffer pool while the encoder begins operating at the new dimensions. Motion estimation then computes offsets into these undersized reference buffers using the new dimensions, reading past the end of the allocation.
The row-MT worker thread (enc_row_mt_worker_hook) continues executing vp9_pick_inter_mode, which calls into SAD (sum of absolute differences) functions against a reference frame buffer that’s already been freed and reallocated for the new dimensions. The SAD function reads a 64x64 block at a stride (rdx=0x360, 864 bytes) calibrated to the old frame width, walking off the end of the new allocation. ASAN catches this as an access-violation READ at a heap address, confirming genuine use-after-free with heap-relative addressing.
By toying with the timing and dimension configurations, we’ve been able to hit multiple crash sites in the VP9 path. The primary crash is in vpx_sad64x64_avx2 (sad_avx2.c:113), but we’ve also triggered vpx_sad16x16_sse2 with smaller dimension transitions. The crash site depends on which block partition size the encoder selects for the current superblock, which varies with the dimension mismatch and content. By carefully selecting the dimensions we have some control over the length of the read (64x64, 32x32, 16x16, 8x, 4x), making it a controlled read.
The VP9 path requires latencyMode: 'realtime' to force threaded tile encoding (vp9_encode_tiles_row_mt), which widens the race window. Rapid dimension changes across superblock/MI allocation boundaries (e.g. 120x120 to 128x128, crossing mi_cols 15 to 16) maximize the likelihood that the internal reallocation changes buffer sizes and strides. The race is reachable from any origin via the WebCodecs VideoEncoder API with no user interaction, permissions, or flags required. Both crashes occur in the renderer process on a thread pool worker thread (T5/T6).
AV1 (libaom): The same root cause also affects the AV1 encoder, which is expected given that libaom’s buffer pool design and encoder architecture originate from its fork of libvpx. The AV1 variant manifests differently: av1_encode_tiles_row_mt calls memset on a buffer pointer that the reconfigure has already nulled out, resulting in a WRITE to 0x000000000000 (rax=0 at crash). Unlike VP9 where libvpx leaves the pointer dangling, libaom’s teardown zeroes it, so it surfaces as a null dereference rather than a UAF. AV1 triggers under default encoder settings without needing latencyMode: 'realtime'.
Reproducer: open the attached HTML, click VP9 or AV1, wait. It calls configure() with random dimensions (1x1 through 800x600) followed by encode() in a loop. On the systems I’ve tested this on the VP9 crash happens roughly after 30 seconds, the AV1 crash is close to instant.
Impact analysis
At minimum this is an controlled out-of-bounds heap read in the renderer process, triggerable from any webpage via the WebCodecs API without user interaction. On most heap layouts the read hits unmapped pages and crashes the tab. When adjacent pages happen to be committed, the read crosses into neighboring allocations, which could theoretically be an information disclosure or ASLR bypass vector, with the right heap massaging and finding an read length that ends up in between the padding of the frame and end of mapped heap pages, though we haven’t been able to demonstrate that from JavaScript.
Since the underlying issue is the encoder operating in an invalid state (mismatched dimensions between active config and retained reference buffers), it’s hard to fully determine the impact. The bug surfaces as an OOB read in VP9 motion estimation, but the same invalid state is present throughout the encode pipeline. We’ve confirmed multiple crash sites in the VP9 path (vpx_sad64x64_avx2, vpx_sad16x16_sse2) depending on timing and dimension configuration, which supports this: the stale reference buffers are reachable from various points in the encoder, and which one faults first depends on the specific encode parameters. The AV1 variant crashing as a write to a low constant address rather than a heap read further supports the idea that the consequences depend on which code path encounters the stale buffers first. Other code paths in the encoder that touch reference frames (reconstruction, loop filtering, etc.) could potentially be reached with different exploitation strategies.
The cause
What version of Chrome have you found the security issue in?
[147.0.7696.0] + [ASAN], [145.0.7632.110] + [stable]
Is the security issue related to a crash?
Yes, it is related to a crash.
Choose the type of vulnerability
Memory Corruption (in a sandboxed process)
How would you like to be publicly acknowledged for your report?
Casper Woudenberg