From 1618174f833943eb9c90733990eb4adc7e46dfd4 Mon Sep 17 00:00:00 2001
From: Le Hoang Quyen <[email protected]>
Date: Fri, 15 May 2026 21:10:01 +0800
Subject: [PATCH] Metal: Fix potential OOB write in Transform Feedback

Fixes a bug where incomplete primitives in transform feedback could
cause incorrect buffer offset calculations, potentially leading to
out-of-bounds writes in the Metal backend. We now round down the
vertex count for independent primitives when XFB is active.

Bug: angleproject:513010645
Change-Id: I82b91500108fb1a7059e43479e7af87702d33b6d
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/7851496
Commit-Queue: Quyen Le <[email protected]>
Reviewed-by: Kenneth Russell <[email protected]>
Commit-Queue: Shahbaz Youssefi <[email protected]>
Reviewed-by: Shahbaz Youssefi <[email protected]>
Auto-Submit: Quyen Le <[email protected]>
Reviewed-by: Geoff Lang <[email protected]>
---

diff --git a/src/libANGLE/TransformFeedback.h b/src/libANGLE/TransformFeedback.h
index 8028d82..d78d6d6 100644
--- a/src/libANGLE/TransformFeedback.h
+++ b/src/libANGLE/TransformFeedback.h
@@ -11,6 +11,7 @@
 
 #include "common/PackedEnums.h"
 #include "common/angleutils.h"
+#include "common/mathutil.h"
 #include "libANGLE/Debug.h"
 
 #include "angle_gl.h"
@@ -29,6 +30,10 @@
 class Context;
 class Program;
 
+angle::CheckedNumeric<GLsizeiptr> GetVerticesNeededForDraw(PrimitiveMode primitiveMode,
+                                                           GLsizei count,
+                                                           GLsizei primcount);
+
 class TransformFeedbackState final : angle::NonCopyable
 {
   public:
diff --git a/src/libANGLE/renderer/metal/ContextMtl.mm b/src/libANGLE/renderer/metal/ContextMtl.mm
index 3fb5453..753e332 100644
--- a/src/libANGLE/renderer/metal/ContextMtl.mm
+++ b/src/libANGLE/renderer/metal/ContextMtl.mm
@@ -450,6 +450,23 @@
     // Real instances count. Zero means this is not instanced draw.
     GLsizei instanceCount = instances ? instances : 1;
 
+    if (mState.isTransformFeedbackActiveUnpaused())
+    {
+        // ES 3.0 requires XFB mode to be points, lines, or triangles.
+        // For this workaround, we assume the draw mode is also one of these.
+        CHECK(mode == gl::PrimitiveMode::Triangles || mode == gl::PrimitiveMode::Lines ||
+              mode == gl::PrimitiveMode::Points);
+
+        // Transform feedback only outputs complete primitives. For independent primitives
+        // (triangles and lines), any leftover vertices do not form a primitive and should
+        // not be processed for transform feedback. Since Metal only supports up to ES 3.0
+        // (where VS cannot have side effects), it is safe to round down the count to avoid
+        // processing them and hitting bugs with incomplete primitives in XFB.
+        // The helper function also returns the correct value for Points mode (no changes).
+        GLsizeiptr verticesNeeded = gl::GetVerticesNeededForDraw(mode, count, 1).ValueOrDie();
+        count                     = static_cast<GLsizei>(verticesNeeded);
+    }
+
     if (mCullAllPolygons && gl::IsPolygonMode(mode))
     {
         return angle::Result::Continue;
diff --git a/src/tests/angle_end2end_tests_expectations.txt b/src/tests/angle_end2end_tests_expectations.txt
index 93de867..41a9ab7 100644
--- a/src/tests/angle_end2end_tests_expectations.txt
+++ b/src/tests/angle_end2end_tests_expectations.txt
@@ -2742,6 +2742,13 @@
 500930576 OPENGL : ReadPixelsTest.LargeTexture/* = SKIP
 // Crashes inside AMD Metal driver when copying the texture -> buffer
 500930576 METAL AMD : ReadPixelsTest.LargeTexture/* = SKIP
+
+// XFB with incomplete primitives fail on GL and Vulkan
+513536751 INTEL OPENGL : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/* = SKIP
+// fails on Vulkan without transform feedback extension
+513536751 VULKAN : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/*NoSupportsTransformFeedbackExtension* = SKIP
+513536751 VULKAN SWIFTSHADER : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/* = SKIP
+
 // Fails test on Metal
 499091328 MAC AMD METAL : Texture2DArrayTestES3.ReformatStagedBufferUpdatesLayerCountOOB/* = SKIP
 
diff --git a/src/tests/gl_tests/TransformFeedbackTest.cpp b/src/tests/gl_tests/TransformFeedbackTest.cpp
index bc13f64..63126cd 100644
--- a/src/tests/gl_tests/TransformFeedbackTest.cpp
+++ b/src/tests/gl_tests/TransformFeedbackTest.cpp
@@ -5273,6 +5273,86 @@
     glFinish();
 }
 
+// Test that transform feedback with instanced drawing and count that results in incomplete
+// primitives works correctly when the buffer is large enough to store all complete primitives from
+// all instances.
+TEST_P(TransformFeedbackTest, InstancedOverflowIncompletePrimitive)
+{
+    // We need ES3 for transform feedback and instancing.
+
+    constexpr char kVS[] = R"(#version 300 es
+        out float out_value;
+        void main() {
+            out_value = float(gl_VertexID) + float(gl_InstanceID) * 100.0;
+            gl_Position = vec4(0.0, 0.0, 0.0, 1.0);
+        })";
+
+    constexpr char kFS[] = R"(#version 300 es
+        out mediump vec4 color;
+        void main() {
+            color = vec4(0.0, 1.0, 0.0, 1.0);
+        })";
+
+    std::vector<std::string> tfVaryings;
+    tfVaryings.push_back("out_value");
+
+    mProgram = CompileProgramWithTransformFeedback(kVS, kFS, tfVaryings, GL_INTERLEAVED_ATTRIBS);
+    ASSERT_NE(0u, mProgram);
+
+    glUseProgram(mProgram);
+
+    // Buffer size: enough for 18 vertices (3 triangles * 3 vertices * 2 instances).
+    // Each vertex writes 1 float.
+    const size_t kBufferSize = 18 * sizeof(float);
+
+    GLBuffer tfBuffer;
+    glBindBuffer(GL_TRANSFORM_FEEDBACK_BUFFER, tfBuffer);
+    glBufferData(GL_TRANSFORM_FEEDBACK_BUFFER, kBufferSize, nullptr, GL_STATIC_DRAW);
+    glBindBufferBase(GL_TRANSFORM_FEEDBACK_BUFFER, 0, tfBuffer);
+
+    GLQuery primitivesWrittenQuery;
+    glBeginQuery(GL_TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN, primitivesWrittenQuery);
+
+    glBeginTransformFeedback(GL_TRIANGLES);
+
+    // Draw 11 vertices (3 triangles and 2 vertices), 2 instances.
+    // Total expected vertices if buffer was large enough:
+    // Instance 0: 9 vertices (3 triangles) + 2 vertices (incomplete) -> only 9 written.
+    // Instance 1: 9 vertices (3 triangles) + 2 vertices (incomplete) -> only 9 written.
+    glDrawArraysInstanced(GL_TRIANGLES, 0, 11, 2);
+
+    glEndTransformFeedback();
+    glEndQuery(GL_TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN);
+    ASSERT_GL_NO_ERROR();
+
+    // Map buffer and check results.
+    void *mappedBuffer =
+        glMapBufferRange(GL_TRANSFORM_FEEDBACK_BUFFER, 0, kBufferSize, GL_MAP_READ_BIT);
+    ASSERT_NE(nullptr, mappedBuffer);
+
+    float *mappedFloats = static_cast<float *>(mappedBuffer);
+
+    // Instance 0 should fill the first 9 floats with values 0 to 8.
+    for (unsigned int i = 0; i < 9; ++i)
+    {
+        EXPECT_EQ(mappedFloats[i], float(i)) << "At index " << i;
+    }
+
+    // Instance 1 should fill the next 9 floats with values 100 to 108.
+    for (unsigned int i = 0; i < 9; ++i)
+    {
+        EXPECT_EQ(mappedFloats[9 + i], 100.0f + float(i)) << "At index " << (9 + i);
+    }
+
+    glUnmapBuffer(GL_TRANSFORM_FEEDBACK_BUFFER);
+
+    // Check primitives written query result.
+    GLuint primitivesWritten = 0;
+    glGetQueryObjectuiv(primitivesWrittenQuery, GL_QUERY_RESULT_EXT, &primitivesWritten);
+    // 3 triangles from instance 0 + 3 triangles from instance 1 = 6 triangles.
+    EXPECT_EQ(6u, primitivesWritten);
+}
+
 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TransformFeedbackTest);
 ANGLE_INSTANTIATE_TEST_ES3_AND(TransformFeedbackTest,
                                ES3_VULKAN().disable(Feature::SupportsTransformFeedbackExtension),

diff --git a/src/tests/angle_end2end_tests_expectations.txt b/src/tests/angle_end2end_tests_expectations.txt
index 93de867..41a9ab7 100644
--- a/src/tests/angle_end2end_tests_expectations.txt
+++ b/src/tests/angle_end2end_tests_expectations.txt
@@ -2742,6 +2742,13 @@
 500930576 OPENGL : ReadPixelsTest.LargeTexture/* = SKIP
 // Crashes inside AMD Metal driver when copying the texture -> buffer
 500930576 METAL AMD : ReadPixelsTest.LargeTexture/* = SKIP
+
+// XFB with incomplete primitives fail on GL and Vulkan
+513536751 INTEL OPENGL : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/* = SKIP
+// fails on Vulkan without transform feedback extension
+513536751 VULKAN : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/*NoSupportsTransformFeedbackExtension* = SKIP
+513536751 VULKAN SWIFTSHADER : TransformFeedbackTest.InstancedOverflowIncompletePrimitive/* = SKIP
+
 // Fails test on Metal
 499091328 MAC AMD METAL : Texture2DArrayTestES3.ReformatStagedBufferUpdatesLayerCountOOB/* = SKIP
diff --git a/src/tests/gl_tests/TransformFeedbackTest.cpp b/src/tests/gl_tests/TransformFeedbackTest.cpp
index bc13f64..63126cd 100644
--- a/src/tests/gl_tests/TransformFeedbackTest.cpp
+++ b/src/tests/gl_tests/TransformFeedbackTest.cpp
@@ -5273,6 +5273,86 @@
     glFinish();
 }
 
+// Test that transform feedback with instanced drawing and count that results in incomplete
+// primitives works correctly when the buffer is large enough to store all complete primitives from
+// all instances.
+TEST_P(TransformFeedbackTest, InstancedOverflowIncompletePrimitive)
+{
+    // We need ES3 for transform feedback and instancing.
+
+    constexpr char kVS[] = R"(#version 300 es
+        out float out_value;
+        void main() {
+            out_value = float(gl_VertexID) + float(gl_InstanceID) * 100.0;
+            gl_Position = vec4(0.0, 0.0, 0.0, 1.0);
+        })";
+
+    constexpr char kFS[] = R"(#version 300 es
+        out mediump vec4 color;
+        void main() {
+            color = vec4(0.0, 1.0, 0.0, 1.0);
+        })";
+
+    std::vector<std::string> tfVaryings;
+    tfVaryings.push_back("out_value");
+
+    mProgram = CompileProgramWithTransformFeedback(kVS, kFS, tfVaryings, GL_INTERLEAVED_ATTRIBS);
+    ASSERT_NE(0u, mProgram);
+
+    glUseProgram(mProgram);
+
+    // Buffer size: enough for 18 vertices (3 triangles * 3 vertices * 2 instances).
+    // Each vertex writes 1 float.
+    const size_t kBufferSize = 18 * sizeof(float);
+
+    GLBuffer tfBuffer;
+    glBindBuffer(GL_TRANSFORM_FEEDBACK_BUFFER, tfBuffer);
+    glBufferData(GL_TRANSFORM_FEEDBACK_BUFFER, kBufferSize, nullptr, GL_STATIC_DRAW);
+    glBindBufferBase(GL_TRANSFORM_FEEDBACK_BUFFER, 0, tfBuffer);
+
+    GLQuery primitivesWrittenQuery;
+    glBeginQuery(GL_TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN, primitivesWrittenQuery);
+
+    glBeginTransformFeedback(GL_TRIANGLES);
+
+    // Draw 11 vertices (3 triangles and 2 vertices), 2 instances.
+    // Total expected vertices if buffer was large enough:
+    // Instance 0: 9 vertices (3 triangles) + 2 vertices (incomplete) -> only 9 written.
+    // Instance 1: 9 vertices (3 triangles) + 2 vertices (incomplete) -> only 9 written.
+    glDrawArraysInstanced(GL_TRIANGLES, 0, 11, 2);
+
+    glEndTransformFeedback();
+    glEndQuery(GL_TRANSFORM_FEEDBACK_PRIMITIVES_WRITTEN);
+    ASSERT_GL_NO_ERROR();
+
+    // Map buffer and check results.
+    void *mappedBuffer =
+        glMapBufferRange(GL_TRANSFORM_FEEDBACK_BUFFER, 0, kBufferSize, GL_MAP_READ_BIT);
+    ASSERT_NE(nullptr, mappedBuffer);
+
+    float *mappedFloats = static_cast<float *>(mappedBuffer);
+
+    // Instance 0 should fill the first 9 floats with values 0 to 8.
+    for (unsigned int i = 0; i < 9; ++i)
+    {
+        EXPECT_EQ(mappedFloats[i], float(i)) << "At index " << i;
+    }
+
+    // Instance 1 should fill the next 9 floats with values 100 to 108.
+    for (unsigned int i = 0; i < 9; ++i)
+    {
+        EXPECT_EQ(mappedFloats[9 + i], 100.0f + float(i)) << "At index " << (9 + i);
+    }
+
+    glUnmapBuffer(GL_TRANSFORM_FEEDBACK_BUFFER);
+
+    // Check primitives written query result.
+    GLuint primitivesWritten = 0;
+    glGetQueryObjectuiv(primitivesWrittenQuery, GL_QUERY_RESULT_EXT, &primitivesWritten);
+    // 3 triangles from instance 0 + 3 triangles from instance 1 = 6 triangles.
+    EXPECT_EQ(6u, primitivesWritten);
+}
+
 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TransformFeedbackTest);
 ANGLE_INSTANTIATE_TEST_ES3_AND(TransformFeedbackTest,
                                ES3_VULKAN().disable(Feature::SupportsTransformFeedbackExtension),