From 4b7aace914924f634c4148c41e6fc87198867fa5 Mon Sep 17 00:00:00 2001
From: Stephen White <[email protected]>
Date: Wed, 04 Mar 2026 15:44:30 -0500
Subject: [PATCH] D3D11: fix overflow in line loop and triangle fan

If an index buffer is created of element type less than 32bit (e.g.,
GL_UNSIGNED_BYTE) and used in a line loop or triangle fan, D3D11 must
widen it to 32-bit. This can cause an overflow if the widened size in
bytes is greater than INT_MAX. The fix is to detect the overflow and
abort.

Bug: chromium:487977696
Change-Id: I57b1dcc9b3d968da88282164a5f92386500c6205
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/7637780
Reviewed-by: Geoff Lang <[email protected]>
Commit-Queue: Stephen White <[email protected]>
---

diff --git a/src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp b/src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp
index 60c00fe..ec644eb 100644
--- a/src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp
+++ b/src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp
@@ -2083,8 +2083,14 @@
     {
         return angle::Result::Continue;
     }
-    unsigned int spaceNeeded =
-        static_cast<unsigned int>(sizeof(GLuint) * mScratchIndexDataBuffer.size());
+
+    uint64_t spaceNeeded64 = sizeof(GLuint) * mScratchIndexDataBuffer.size();
+    ANGLE_CHECK(GetImplAs<Context11>(context), spaceNeeded64 <= std::numeric_limits<int>::max(),
+                "Failed to create a 32-bit looping index buffer for "
+                "a GL_LINE_LOOP of <32-bit element type; too many indices required.",
+                GL_OUT_OF_MEMORY);
+    int spaceNeeded = static_cast<int>(spaceNeeded64);
+
     ANGLE_TRY(
         mLineLoopIB->reserveBufferSpace(context, spaceNeeded, gl::DrawElementsType::UnsignedInt));
 
@@ -2171,8 +2177,12 @@
     GetTriFanIndices(indexPointer, type, count, glState.isPrimitiveRestartEnabled(),
                      &mScratchIndexDataBuffer);
 
-    const unsigned int spaceNeeded =
-        static_cast<unsigned int>(mScratchIndexDataBuffer.size() * sizeof(unsigned int));
+    uint64_t spaceNeeded64 = mScratchIndexDataBuffer.size() * sizeof(unsigned int);
+    ANGLE_CHECK(GetImplAs<Context11>(context), spaceNeeded64 <= std::numeric_limits<int>::max(),
+                "Failed to create a 32-bit looping index buffer for "
+                "a GL_TRIANGLE_FAN of <32-bit element type; too many indices required.",
+                GL_OUT_OF_MEMORY);
+    int spaceNeeded = static_cast<int>(spaceNeeded64);
     ANGLE_TRY(mTriangleFanIB->reserveBufferSpace(context, spaceNeeded,
                                                  gl::DrawElementsType::UnsignedInt));
 
diff --git a/src/tests/gl_tests/LineLoopTest.cpp b/src/tests/gl_tests/LineLoopTest.cpp
index 696dde3..2dc6551 100644
--- a/src/tests/gl_tests/LineLoopTest.cpp
+++ b/src/tests/gl_tests/LineLoopTest.cpp
@@ -546,6 +546,46 @@
     EXPECT_PIXEL_COLOR_EQ(0, 0, GLColor::green);
 }
 
+// Test an edge case in the D3D11 backend where a large index array of GL_UNSIGNED_BYTE
+// drawn as a GL_LINE_LOOP is widened to 32-bits internally and overflows.
+// Disabled because it slow and triggers an internal error.
+TEST_P(LineLoopTestES3, DISABLED_LargeLoopUnsignedByte)
+{
+    size_t count = 805306368;
+
+    glLinkProgram(mProgram);
+    glUseProgram(mProgram);
+    ASSERT_GL_NO_ERROR();
+
+    GLBuffer ebo;
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, count, nullptr, GL_STATIC_DRAW);
+    ASSERT_GL_NO_ERROR();
+
+    glDrawElements(GL_LINE_LOOP, count, GL_UNSIGNED_BYTE, 0);
+    EXPECT_GL_ERROR(GL_OUT_OF_MEMORY);
+}
+
+// Test an edge case in the D3D11 backend where a large index array of GL_UNSIGNED_BYTE
+// drawn as a GL_TRIANGLE_FAN is widened to 32-bits internally and overflows.
+// Disabled because it slow and triggers an internal error.
+TEST_P(LineLoopTestES3, DISABLED_LargeFanUnsignedByte)
+{
+    size_t count = 805306368;
+
+    glLinkProgram(mProgram);
+    glUseProgram(mProgram);
+    ASSERT_GL_NO_ERROR();
+
+    GLBuffer ebo;
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, count, nullptr, GL_STATIC_DRAW);
+    ASSERT_GL_NO_ERROR();
+
+    glDrawElements(GL_TRIANGLE_FAN, count, GL_UNSIGNED_BYTE, 0);
+    EXPECT_GL_ERROR(GL_OUT_OF_MEMORY);
+}
+
 // Tests an edge case with a very large line loop element count.
 // Disabled because it is slow and triggers an internal error.
 TEST_P(LineLoopTest, DISABLED_DrawArraysWithLargeCount)

diff --git a/src/tests/gl_tests/LineLoopTest.cpp b/src/tests/gl_tests/LineLoopTest.cpp
index 696dde3..2dc6551 100644
--- a/src/tests/gl_tests/LineLoopTest.cpp
+++ b/src/tests/gl_tests/LineLoopTest.cpp
@@ -546,6 +546,46 @@
     EXPECT_PIXEL_COLOR_EQ(0, 0, GLColor::green);
 }
 
+// Test an edge case in the D3D11 backend where a large index array of GL_UNSIGNED_BYTE
+// drawn as a GL_LINE_LOOP is widened to 32-bits internally and overflows.
+// Disabled because it slow and triggers an internal error.
+TEST_P(LineLoopTestES3, DISABLED_LargeLoopUnsignedByte)
+{
+    size_t count = 805306368;
+
+    glLinkProgram(mProgram);
+    glUseProgram(mProgram);
+    ASSERT_GL_NO_ERROR();
+
+    GLBuffer ebo;
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, count, nullptr, GL_STATIC_DRAW);
+    ASSERT_GL_NO_ERROR();
+
+    glDrawElements(GL_LINE_LOOP, count, GL_UNSIGNED_BYTE, 0);
+    EXPECT_GL_ERROR(GL_OUT_OF_MEMORY);
+}
+
+// Test an edge case in the D3D11 backend where a large index array of GL_UNSIGNED_BYTE
+// drawn as a GL_TRIANGLE_FAN is widened to 32-bits internally and overflows.
+// Disabled because it slow and triggers an internal error.
+TEST_P(LineLoopTestES3, DISABLED_LargeFanUnsignedByte)
+{
+    size_t count = 805306368;
+
+    glLinkProgram(mProgram);
+    glUseProgram(mProgram);
+    ASSERT_GL_NO_ERROR();
+
+    GLBuffer ebo;
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, count, nullptr, GL_STATIC_DRAW);
+    ASSERT_GL_NO_ERROR();
+
+    glDrawElements(GL_TRIANGLE_FAN, count, GL_UNSIGNED_BYTE, 0);
+    EXPECT_GL_ERROR(GL_OUT_OF_MEMORY);
+}
+
 // Tests an edge case with a very large line loop element count.
 // Disabled because it is slow and triggers an internal error.
 TEST_P(LineLoopTest, DISABLED_DrawArraysWithLargeCount)