Use stencil test to not draw floors where there are chasms

shader/lighting/wallShadow.frag

@@ -1,6 +0,0 @@
-#version 450 core
-out vec4 fColor;
-void main()
-{
-  fColor = vec4 (0,0,0,1);
-}

shader/lighting/wallShadow.geom

@@ -36,8 +36,8 @@ void main() {
     vec2 p2cs = p2 + closeshift;
     vec2 p2o = vec2( distance(p2,lightPos.xy) > rad2 ? p2cs : shift(p2) );
 
-    gl_Position = theMat * vec4(p1o,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p1o,-500,1); EmitVertex();
-    gl_Position = theMat * vec4(p1,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p1,-500,1); EmitVertex();
     gl_Position = theMat * vec4(p1o,100,1); EmitVertex();
     gl_Position = theMat * vec4(p1,100,1); EmitVertex();
     gl_Position = theMat * vec4(p1cs,100,1); EmitVertex();
@@ -52,15 +52,15 @@ void main() {
 //    gl_Position = theMat * vec4(p2,0,1); EmitVertex();
     
     gl_Position = theMat * vec4(p2,100,1); EmitVertex();
-    gl_Position = theMat * vec4(p2,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p2,-500,1); EmitVertex();
     gl_Position = theMat * vec4(p2o,100,1); EmitVertex();
-    gl_Position = theMat * vec4(p2o,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p2o,-500,1); EmitVertex();
     gl_Position = theMat * vec4(p2cs,100,1); EmitVertex();
-    gl_Position = theMat * vec4(p2cs,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p2cs,-500,1); EmitVertex();
     gl_Position = theMat * vec4(p1cs,100,1); EmitVertex();
-    gl_Position = theMat * vec4(p1cs,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p1cs,-500,1); EmitVertex();
     gl_Position = theMat * vec4(p1o,100,1); EmitVertex();
-    gl_Position = theMat * vec4(p1o,0,1); EmitVertex();
+    gl_Position = theMat * vec4(p1o,-500,1); EmitVertex();
     
     EndPrimitive();
   } else {

src/Dodge/Render.hs

@@ -1,24 +1,23 @@
 -- | Contains the central drawing functions for the dodge loop.
 module Dodge.Render (
     doDrawing,
-    getWindowSize
+    getWindowSize,
 ) where
 
-import qualified Data.Vector as V
-
-import Dodge.Shadows
-import Dodge.WindowSize
 import Control.Lens
 import Control.Monad
 import qualified Control.Monad.Parallel as MP
 --import Data.List (sortOn)
 import qualified Data.Map.Strict as M
 import Data.Preload.Render
+import qualified Data.Vector as V
 import qualified Data.Vector.Unboxed.Mutable as UMV
 import Dodge.Data.Universe
 import Dodge.Render.Lights
 import Dodge.Render.ShapePicture
 import Dodge.Render.Walls
+import Dodge.Shadows
+import Dodge.WindowSize
 import Foreign
 import GLHelp
 import Geometry
@@ -80,7 +79,8 @@ doDrawing' win pdata u = do
                 windowPoints
                 wallsToPoke
             )
-            ( pokeShape (drawShadowsByImportance $ cfig ^. graphics_shadow_size)
+            ( pokeShape
+                (drawShadowsByImportance $ cfig ^. graphics_shadow_size)
                 (_vboPtr $ _vboShapes pdata)
                 (_eboPtr $ _shapeEBO pdata)
                 (_eboPtr $ _silhouetteEBO pdata)
@@ -88,19 +88,18 @@ doDrawing' win pdata u = do
                 ws
             )
     (nCloudVs', nCloudIs') <-
-    -- Might want to sort the clouds based on z depth, not sure
+        -- Might want to sort the clouds based on z depth, not sure
         V.foldM'
             (pokeCloud (pdata ^. cloudVBO . vboPtr) (pdata ^. cloudEBO . eboPtr))
             (0, 0)
             (V.fromList $ w ^. cWorld . lWorld . clouds)
+    --(sortOn (^. clPos . _3) $ w ^. cWorld . lWorld . clouds)
     (nCloudVs, nCloudIs) <-
-    -- Might want to sort the clouds based on z depth, not sure
         V.foldM'
             (pokeDust (pdata ^. cloudVBO . vboPtr) (pdata ^. cloudEBO . eboPtr))
             (nCloudVs', nCloudIs')
             (V.fromList $ w ^. cWorld . lWorld . dusts)
-    
+
-            --(sortOn (^. clPos . _3) $ w ^. cWorld . lWorld . clouds)
     -- bind wall points, silhouette data, surface geometry
     bufferShaderLayers shadV pokeCounts
     mapM_
@@ -161,6 +160,8 @@ doDrawing' win pdata u = do
             GL_COLOR
             2
             ptr
+    withArray [GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2] $
+        \ptr -> glDrawBuffers 3 ptr
     --draw walls onto base buffer
     -- maybe cull faces? maybe do this before?
     glEnable GL_CULL_FACE
@@ -172,7 +173,7 @@ doDrawing' win pdata u = do
         0
         (fromIntegral trueNWalls)
     --draw object pictures onto base buffer (are there any?!)
---    renderLayer BottomLayer shadV layerCounts
+    --    renderLayer BottomLayer shadV layerCounts
     --draw object shapes onto base buffer
     let fs = _shapeShader pdata
     glUseProgram (_shaderUINT fs)
@@ -185,22 +186,33 @@ doDrawing' win pdata u = do
     -- draw chasm shader blocking floor
     glUseProgram (pdata ^. chasmShader . shaderUINT)
     glBindVertexArray $ pdata ^. chasmShader . shaderVAO . vaoName
---    glColorMask GL_FALSE GL_FALSE GL_FALSE GL_FALSE
+    glDepthMask GL_FALSE
     glDrawBuffer GL_NONE
+    glEnable GL_STENCIL_TEST
+    glStencilFunc GL_ALWAYS 0 255
+    glStencilOp GL_INCR GL_INCR GL_INCR
+    with 0 $ \ptr ->
+        glClearNamedFramebufferiv
+            (pdata ^. fboBase . _1 . unFBO)
+            GL_STENCIL
+            0
+            ptr
     glDrawArrays
         (_unPrimitiveMode $ pdata ^. chasmShader . shaderPrimitive)
         0
         (fromIntegral nchs)
---    glColorMask GL_TRUE GL_TRUE GL_TRUE GL_TRUE
+    withArray [GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2] $
-    withArray [GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2] 
+        \ptr -> glDrawBuffers 3 ptr
-            $ \ptr -> glDrawBuffers 3 ptr
+    glDepthMask GL_TRUE
     --draw floor onto base buffer
     glUseProgram (pdata ^. floorShader . shaderUINT)
     glBindVertexArray $ pdata ^. floorShader . shaderVAO . vaoName
+    glStencilFunc GL_EQUAL 0 255
     glDrawArrays
         (_unPrimitiveMode $ pdata ^. floorShader . shaderPrimitive)
         0
         (fromIntegral nFls)
+    glDisable GL_STENCIL_TEST
     glDisable GL_CULL_FACE
     glEnable GL_BLEND
     --draw lightmap into its own buffer
@@ -245,7 +257,7 @@ doDrawing' win pdata u = do
     -- draw clouds, transparency blocks some alpha
     glDepthMask GL_FALSE
     glBlendColor 0 0 0 0.1 -- using this and GL_CONSTANT_ALPHA below
-                           -- can allow for some underlighting of the clouds
+    -- can allow for some underlighting of the clouds
     glBlendFuncSeparate GL_ZERO GL_ONE GL_CONSTANT_ALPHA GL_ONE_MINUS_SRC_ALPHA
     glUseProgram (pdata ^. cloudShader . shaderUINT)
     glBindVertexArray $ pdata ^. cloudShader . shaderVAO . vaoName
@@ -264,7 +276,7 @@ doDrawing' win pdata u = do
     glBindTextureUnit 0 (pdata ^. fboBloom . _2 . unTO)
     glDisable GL_BLEND
     drawShader (_bloomBlurShader pdata) 4
-    when (cfig ^. graphics_bloom) $ do   
+    when (cfig ^. graphics_bloom) $ do
         replicateM_ 2 $ pingPongBetween (_fboHalf1 pdata) (_fboHalf2 pdata) (_bloomBlurShader pdata)
         glEnable GL_BLEND
     setViewport _graphics_world_resolution cfig
@@ -287,7 +299,7 @@ doDrawing' win pdata u = do
             GL_COLOR
             0
             ptr
---    renderLayer MidLayer shadV pokeCounts
+    --    renderLayer MidLayer shadV pokeCounts
     glUseProgram (pdata ^. cloudShader . shaderUINT)
     glBindVertexArray $ pdata ^. cloudShader . shaderVAO . vaoName
     glDrawElements
@@ -301,8 +313,8 @@ doDrawing' win pdata u = do
         glDepthMask GL_TRUE
         glDepthFunc GL_ALWAYS
         glDisable GL_BLEND
-        withArray [GL_NONE, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2] 
+        withArray [GL_NONE, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2] $
-            $ \ptr -> glDrawBuffers 3 ptr
+            \ptr -> glDrawBuffers 3 ptr
         withArray [0, 0, 0, 0] $ \ptr ->
             glClearNamedFramebufferfv
                 (pdata ^. fboCloud . _1 . unFBO)
@@ -379,7 +391,6 @@ doDrawing' win pdata u = do
     drawShader (_fullscreenShader pdata) 4
     glBlendFunc GL_SRC_ALPHA GL_ONE_MINUS_SRC_ALPHA
 
-
     --set viewport for radial distortion
     --setViewportSize (round winx) (round winy)
     setViewport (const FullRes) cfig
@@ -392,7 +403,7 @@ doDrawing' win pdata u = do
             glBindTextureUnit 0 $ pdata ^. fboBase . _2 . _1 . unTO
             glBindFramebuffer GL_FRAMEBUFFER 0
             drawShader (_fullscreenShader pdata) 4
-        (rds,_) -> do
+        (rds, _) -> do
             let bindDrawDist :: Distortion -> IO ()
                 bindDrawDist (RadialDistortion (V2 a b) (V2 c d) (V2 e f) g) = do
                     pokeArray (shadVBOptr $ _barrelShader pdata) [a, b, c, d, e, f, g]
@@ -413,8 +424,8 @@ doDrawing' win pdata u = do
     glBlendFunc GL_SRC_ALPHA GL_ONE_MINUS_SRC_ALPHA
     renderLayer DebugLayer shadV pokeCounts
     glBindBufferBase GL_UNIFORM_BUFFER 0 (pdata ^. matUBO)
-    withArray (scaleMatrix (2 / windowXFloat cfig) (2 / windowYFloat cfig))
+    withArray (scaleMatrix (2 / windowXFloat cfig) (2 / windowYFloat cfig)) $
-        $ \ptr -> glNamedBufferSubData (pdata ^. matUBO) 0 64 ptr
+        \ptr -> glNamedBufferSubData (pdata ^. matUBO) 0 64 ptr
     glDepthFunc GL_GEQUAL
     glDepthMask GL_TRUE
     with (-1) $ \ptr -> glClearNamedFramebufferfv 0 GL_DEPTH 0 ptr
@@ -422,7 +433,7 @@ doDrawing' win pdata u = do
     SDL.glSwapWindow win
 
 getDistortions :: Configuration -> World -> [Distortion]
-getDistortions cfig w 
+getDistortions cfig w
     | cfig ^. graphics_distortions = w ^. cWorld . lWorld . distortions
     | otherwise = []
 

src/Render.hs

@@ -153,6 +153,7 @@ renderShadows shadrendertype nWalls nSils nCaps positiontexture normaltexture li
     -- 1. stencil out the shadows from this light's point of view
     -- 2. calculate lighting based on each fragment's position and normal
     flip VFSM.mapM_ (VFSM.fromList lightPoints) $ \(V3 x y z, rad, V3 r g b) -> do
+        -- note that the stencil shadows rely on the depth buffer
         glDepthFunc GL_LESS
         -- setup stencil
         glColorMask GL_FALSE GL_FALSE GL_FALSE GL_FALSE