Fix additional overflow and underflow issues with triangle fans and line loops.

Issue #444

Signed-off-by: Shannon Woods
Signed-off-by: Chris Evans
Author: Geoff Lang

3 files changed, 30 insertions, 16 deletions

diff --git a/src/common/version.h b/src/common/version.h
index 68cf494b..2e125120 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -1,7 +1,7 @@
 #define MAJOR_VERSION 1
 #define MINOR_VERSION 2
 #define BUILD_VERSION 0
-#define BUILD_REVISION 2429
+#define BUILD_REVISION 2430
 
 #define STRINGIFY(x) #x
 #define MACRO_STRINGIFY(x) STRINGIFY(x)
diff --git a/src/libGLESv2/renderer/Renderer11.cpp b/src/libGLESv2/renderer/Renderer11.cpp
index 07db7de8..9173fc8c 100644
--- a/src/libGLESv2/renderer/Renderer11.cpp
+++ b/src/libGLESv2/renderer/Renderer11.cpp
@@ -835,16 +835,18 @@ bool Renderer11::applyPrimitiveType(GLenum mode, GLsizei count)
 {
     D3D11_PRIMITIVE_TOPOLOGY primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
 
+    GLsizei minCount = 0;
+
     switch (mode)
     {
-      case GL_POINTS:         primitiveTopology = D3D11_PRIMITIVE_TOPOLOGY_POINTLIST;   break;
-      case GL_LINES:          primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINELIST;      break;
-      case GL_LINE_LOOP:      primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP;     break;
-      case GL_LINE_STRIP:     primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP;     break;
-      case GL_TRIANGLES:      primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST;  break;
-      case GL_TRIANGLE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP; break;
+      case GL_POINTS:         primitiveTopology = D3D11_PRIMITIVE_TOPOLOGY_POINTLIST;   minCount = 1; break;
+      case GL_LINES:          primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINELIST;      minCount = 2; break;
+      case GL_LINE_LOOP:      primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP;     minCount = 2; break;
+      case GL_LINE_STRIP:     primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP;     minCount = 2; break;
+      case GL_TRIANGLES:      primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST;  minCount = 3; break;
+      case GL_TRIANGLE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP; minCount = 3; break;
           // emulate fans via rewriting index buffer
-      case GL_TRIANGLE_FAN:   primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST;  break;
+      case GL_TRIANGLE_FAN:   primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST;  minCount = 3; break;
       default:
         return gl::error(GL_INVALID_ENUM, false);
     }
@@ -855,7 +857,7 @@ bool Renderer11::applyPrimitiveType(GLenum mode, GLsizei count)
         mCurrentPrimitiveTopology = primitiveTopology;
     }
 
-    return count > 0;
+    return count >= minCount;
 }
 
 bool Renderer11::applyRenderTarget(gl::Framebuffer *framebuffer)
@@ -1131,13 +1133,16 @@ void Renderer11::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
         }
     }
 
-    if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+    // Checked by Renderer11::applyPrimitiveType
+    ASSERT(count >= 0);
+
+    if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
     {
         ERR("Could not create a 32-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
         return gl::error(GL_OUT_OF_MEMORY);
     }
 
-    const unsigned int spaceNeeded = (count + 1) * sizeof(unsigned int);
+    const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned int);
     if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT))
     {
         ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");
@@ -1231,9 +1236,12 @@ void Renderer11::drawTriangleFan(GLsizei count, GLenum type, const GLvoid *indic
         }
     }
 
+    // Checked by Renderer11::applyPrimitiveType
+    ASSERT(count >= 3);
+
     const unsigned int numTris = count - 2;
 
-    if (numTris * 3 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+    if (numTris > (std::numeric_limits<unsigned int>::max() / (sizeof(unsigned int) * 3)))
     {
         ERR("Could not create a scratch index buffer for GL_TRIANGLE_FAN, too many indices required.");
         return gl::error(GL_OUT_OF_MEMORY);
diff --git a/src/libGLESv2/renderer/Renderer9.cpp b/src/libGLESv2/renderer/Renderer9.cpp
index 4aecd4ae..240367d5 100644
--- a/src/libGLESv2/renderer/Renderer9.cpp
+++ b/src/libGLESv2/renderer/Renderer9.cpp
@@ -1471,13 +1471,16 @@ void Renderer9::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
             }
         }
 
-        if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+        if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
         {
             ERR("Could not create a 32-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
             return gl::error(GL_OUT_OF_MEMORY);
         }
 
-        const unsigned int spaceNeeded = (count + 1) * sizeof(unsigned int);
+        // Checked by Renderer9::applyPrimitiveType
+        ASSERT(count >= 0);
+
+        const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned int);
         if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT))
         {
             ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");
@@ -1549,13 +1552,16 @@ void Renderer9::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
             }
         }
 
-        if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned short>::max() / sizeof(unsigned short)))
+        // Checked by Renderer9::applyPrimitiveType
+        ASSERT(count >= 0);
+
+        if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned short>::max() / sizeof(unsigned short)))
         {
             ERR("Could not create a 16-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
             return gl::error(GL_OUT_OF_MEMORY);
         }
 
-        const int spaceNeeded = (count + 1) * sizeof(unsigned short);
+        const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned short);
         if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT))
         {
             ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");