recursive stencil changes

1 files changed, 71 insertions, 71 deletions

diff --git a/src/gl_calls.cpp b/src/gl_calls.cpp
index 91dba4f..8bdd7f1 100644
--- a/src/gl_calls.cpp
+++ b/src/gl_calls.cpp
@@ -180,17 +180,8 @@ static void
 GL_BlitStencil(gl_effect_layer *TestM, void *StrokeData, void *FillData, uint32 StrokeCount, uint32 FillCount,
                   layer_transforms T, int Width, int Height, int BytesPerPixel, 
                   int L_Width, int L_Height,v4 StrokeCol, v4 FillCol, int RenderMode, int Vector,
-                  ImVec2 ViewportSize, ImVec2 UIPos, ImVec2 UIZoom)
+                  ImVec2 ViewportSize, ImVec2 UIPos, ImVec2 UIZoom, int StencilLayer, int Mode)
 {
-    // stencil buffer
-    glEnable(GL_STENCIL_TEST);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-    glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
-    glClear(GL_STENCIL_BUFFER_BIT);
-    glStencilFunc(GL_ALWAYS, 0, 0xFF);
-    glStencilMask(0xff);
-    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-
     int Uniform = 0;
 
     Uniform = glGetUniformLocation(DefaultShaderProgram, "CompDimensions");
@@ -220,6 +211,11 @@ GL_BlitStencil(gl_effect_layer *TestM, void *StrokeData, void *FillData, uint32
     Uniform = glGetUniformLocation(DefaultShaderProgram, "InputCol");
     glUniform3f(Uniform, 0, 0, 0);
 
+    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+    glBindVertexArray(0);
+    glStencilFunc(GL_EQUAL, StencilLayer, 0xFF);
+    glStencilOp(GL_KEEP, GL_KEEP, Mode);
+
     // fill
         // vertices
         glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
@@ -228,13 +224,9 @@ GL_BlitStencil(gl_effect_layer *TestM, void *StrokeData, void *FillData, uint32
         glEnableVertexAttribArray(0);
         glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
 
-        // stencil buffer
-	    glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_INCR_WRAP);
-	    glStencilOpSeparate(GL_BACK, GL_KEEP, GL_KEEP, GL_DECR_WRAP);
-	    glDisable(GL_CULL_FACE);
-
         glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_INT, 0);
     //
+
 	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     glBindVertexArray(0);
 
@@ -244,18 +236,8 @@ static void
 GL_RasterizeShape2(gl_effect_layer *TestM, void *StrokeData, void *FillData, uint32 StrokeCount, uint32 FillCount,
                   layer_transforms T, int Width, int Height, int BytesPerPixel, 
                   int L_Width, int L_Height,v4 StrokeCol, v4 FillCol, int RenderMode, int Vector,
-                  ImVec2 ViewportSize, ImVec2 UIPos, ImVec2 UIZoom, bool32 KeepStencil)
+                  ImVec2 ViewportSize, ImVec2 UIPos, ImVec2 UIZoom, int StencilLayer)
 {
-    // stencil buffer
-    glEnable(GL_STENCIL_TEST);
-    glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-    glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
-    if (!KeepStencil)
-        glClear(GL_STENCIL_BUFFER_BIT);
-    glStencilFunc(GL_ALWAYS, 0, 0xFF);
-    glStencilMask(0xff);
-    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-
     int Uniform = 0;
 
     Uniform = glGetUniformLocation(DefaultShaderProgram, "CompDimensions");
@@ -280,30 +262,34 @@ GL_RasterizeShape2(gl_effect_layer *TestM, void *StrokeData, void *FillData, uin
 
     if (RenderMode == 0 || RenderMode == 1)
     {
+        // disable color component writing and allow stencil writing using the shape layer's vertices
+        glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+        glBindVertexArray(0);
+        glStencilFunc(GL_ALWAYS, 0, 0xFF);
+
+	    glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_INCR_WRAP);
+	    glStencilOpSeparate(GL_BACK,  GL_KEEP, GL_KEEP, GL_DECR_WRAP);
 
         Uniform = glGetUniformLocation(DefaultShaderProgram, "VertexMode");
         glUniform1i(Uniform, 1);
         Uniform = glGetUniformLocation(DefaultShaderProgram, "FragmentMode");
         glUniform1i(Uniform, 1);
 
-        // fill
-            // vertices
-            glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
-            glBufferData(GL_ARRAY_BUFFER, sizeof(real32) * 4 * FillCount, FillData, GL_STATIC_DRAW);
-            glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
-            glEnableVertexAttribArray(0);
-            glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
+        glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
+        glBufferData(GL_ARRAY_BUFFER, sizeof(real32) * 4 * FillCount, FillData, GL_STATIC_DRAW);
+        glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
+        glEnableVertexAttribArray(0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
 
-            // stencil buffer
-	        glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_INCR_WRAP);
-	        glStencilOpSeparate(GL_BACK, GL_KEEP, GL_KEEP,  GL_DECR_WRAP);
-	        glDisable(GL_CULL_FACE);
+	    glDrawArrays(GL_TRIANGLE_FAN, 0, FillCount);
 
-	        glDrawArrays(GL_TRIANGLE_FAN, 0, FillCount);
-        //
+        // ---
 
+        // allow color component writing 
 	    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
         glBindVertexArray(0);
+        glStencilFunc(GL_EQUAL, StencilLayer + 1, 0xFF);
+	    glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
 
         Uniform = glGetUniformLocation(DefaultShaderProgram, "VertexMode");
         glUniform1i(Uniform, 0);
@@ -312,33 +298,50 @@ GL_RasterizeShape2(gl_effect_layer *TestM, void *StrokeData, void *FillData, uin
         Uniform = glGetUniformLocation(DefaultShaderProgram, "InputCol");
         glUniform3f(Uniform, FillCol.r, FillCol.g, FillCol.b);
 
-        // stencil for fill
-        //
-            // vertices
-            glBindBuffer(GL_ARRAY_BUFFER, DefaultVerts.VertexBufferObject);
-            glBufferData(GL_ARRAY_BUFFER, sizeof(GL_DefaultVertices), GL_DefaultVertices, GL_STATIC_DRAW);
-            glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)0);
-            glEnableVertexAttribArray(0);
-            glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)(3 * sizeof(float)));
-            glEnableVertexAttribArray(1);
-
-            // stencil buffer
-            glStencilFunc(GL_NOTEQUAL, 0, 0xFF);
-	        glStencilOp(GL_ZERO, GL_ZERO, GL_ZERO);
-
-            glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_INT, 0);
-        //
+        glBindBuffer(GL_ARRAY_BUFFER, DefaultVerts.VertexBufferObject);
+        glBufferData(GL_ARRAY_BUFFER, sizeof(GL_DefaultVertices), GL_DefaultVertices, GL_STATIC_DRAW);
+        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)0);
+        glEnableVertexAttribArray(0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)(3 * sizeof(float)));
+        glEnableVertexAttribArray(1);
+
+        glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_INT, 0);
+
+        // ---
+
+        // NOTE(fox): We need to clean up the mask after we draw the colors if
+        // precomps are involved. 
+        // TODO(fox): If this is the final method, add this optimization: shapes with
+        // no precomps below them don't have to be drawn twice and can instead
+        // get cleaned by setting StencilOp to GL_DECR. 
+
+        glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+        glBindVertexArray(0);
+        glStencilFunc(GL_ALWAYS, 0, 0xFF);
+
+	    glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_DECR_WRAP);
+	    glStencilOpSeparate(GL_BACK,  GL_KEEP, GL_KEEP, GL_INCR_WRAP);
+
+        Uniform = glGetUniformLocation(DefaultShaderProgram, "VertexMode");
+        glUniform1i(Uniform, 1);
+        Uniform = glGetUniformLocation(DefaultShaderProgram, "FragmentMode");
+        glUniform1i(Uniform, 1);
+
+        glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
+        glBufferData(GL_ARRAY_BUFFER, sizeof(real32) * 4 * FillCount, FillData, GL_STATIC_DRAW);
+        glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
+        glEnableVertexAttribArray(0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
+
+	    glDrawArrays(GL_TRIANGLE_FAN, 0, FillCount);
+	    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     } else {
 	    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     }
 
-    // stencil buffer not needed
-	glDisable(GL_STENCIL_TEST);
-    glStencilMask(0xFF);
-    glStencilFunc(GL_ALWAYS, 0, 0xFF);
-
     glBindVertexArray(0);
 
+    // stroke component
     if (RenderMode == 0 || RenderMode == 2) {
         Uniform = glGetUniformLocation(DefaultShaderProgram, "VertexMode");
         glUniform1i(Uniform, 1);
@@ -347,17 +350,14 @@ GL_RasterizeShape2(gl_effect_layer *TestM, void *StrokeData, void *FillData, uin
         Uniform = glGetUniformLocation(DefaultShaderProgram, "InputCol");
         glUniform3f(Uniform, StrokeCol.r, StrokeCol.g, StrokeCol.b);
 
-        // stroke
-        //
-            glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
-            glBufferData(GL_ARRAY_BUFFER, sizeof(real32) * 4 * StrokeCount, StrokeData, GL_STATIC_DRAW);
-            glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
-            glEnableVertexAttribArray(0);
-            glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
-            glEnableVertexAttribArray(1);
-
-	        glDrawArrays(GL_TRIANGLE_STRIP, 0, StrokeCount);
-        //
+        glBindBuffer(GL_ARRAY_BUFFER, ShapeVerts.VertexBufferObject);
+        glBufferData(GL_ARRAY_BUFFER, sizeof(real32) * 4 * StrokeCount, StrokeData, GL_STATIC_DRAW);
+        glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
+        glEnableVertexAttribArray(0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof(float)));
+        glEnableVertexAttribArray(1);
+
+        glDrawArrays(GL_TRIANGLE_STRIP, 0, StrokeCount);
     }
 }