Shadow Mapping w/ FBO & GLSL

I’ve run into a problem with my 3D engine while trying to incorporate shadow mapping.

Vert Shader:
<div class=“ubbcode-block”><div class=“ubbcode-header”>Click to reveal… <input type=“button” class=“form-button” value=“Show me!” onclick=“toggle_spoiler(this, ‘Yikes, my eyes!’, ‘Show me!’)” />]<div style=“display: none;”>

varying float heightPass;

varying vec3 n, eyeVec, lightVec;
varying vec3 HalfVector;
varying vec4 shadowTexCoord;



void main()
{
	n = normalize(gl_NormalMatrix * gl_Normal);
	vec4 vVertex = gl_ModelViewMatrix * gl_Vertex;
	eyeVec = -vVertex.xyz;
	lightVec = normalize(vec3(gl_LightSource[0].position.xyz));
	HalfVector = normalize(gl_LightSource[0].halfVector.xyz);
	
	heightPass = gl_Vertex.y;
	
	
	gl_ClipVertex=gl_ModelViewMatrix*gl_Vertex;

	gl_TexCoord[0] = gl_MultiTexCoord0 * (12); // 6 = tiling factor, makes terrains look better
	gl_TexCoord[1] = gl_MultiTexCoord0 * (6);
	gl_TexCoord[2] = gl_MultiTexCoord0;
	shadowTexCoord = gl_TextureMatrix[7] * gl_Vertex;

	
	gl_Position = ftransform();
	gl_FrontColor = gl_Color;
}

[/QUOTE]
Frag Shader:

uniform sampler2D grass; // 1
uniform sampler2D sand; // 2
uniform sampler2D mud; // 3
uniform sampler2D snow; // 4
uniform sampler2D gravel; // 5
uniform sampler2D detail ; // 6
uniform sampler2D shadowMap;//7

varying float heightPass;
varying vec3 n, lightVec, eyeVec;
varying vec2 texCoord;
varying vec4 pos;
varying vec3 HalfVector;
varying vec4 shadowTexCoord;

float texfactor(float h1, float h2)
{
  float percent;
  percent = (32 - abs(h1 - h2)) / 32;
 
  if(percent < 0.0) percent = 0.0;
  else if(percent > 1.0) percent = 1.0;

  return percent;
}
	float percent[9];
vec4 blendedTerrain(vec4 texel)
{
	
	percent[0] = texfactor(196,heightPass);
	percent[1] = texfactor(164,heightPass);
	percent[2] = texfactor(128,heightPass);
	percent[3] = texfactor(96,heightPass);
	percent[4] = texfactor(64,heightPass);
	percent[5] = texfactor(32,heightPass);
	percent[6] = texfactor(0,heightPass);
	percent[7] = texfactor(228,heightPass);
	percent[8] = texfactor(256,heightPass);
	
	// Fix the order on these things ?
	texel = texture2D(snow,gl_TexCoord[0].st)*percent[0] + // snow starts
		texture2D(grass,gl_TexCoord[0].st)*percent[1]+
		texture2D(grass,gl_TexCoord[0].st)*percent[2] +
		texture2D(grass,gl_TexCoord[0].st)*percent[3] +
		texture2D(sand,gl_TexCoord[0].st)*percent[4] +
		texture2D(sand,gl_TexCoord[0].st)*percent[5] +
		texture2D(mud,gl_TexCoord[0].st)*percent[6] +
		texture2D(snow,gl_TexCoord[0].st)*percent[7]+ // higher
		texture2D(snow,gl_TexCoord[0].st)*percent[8]; // highest
	return texel;
}
void main()
{
	
    //FOG
	float density = 0.1;
	vec4 fogColor = vec4(0.5,0.5,0.5,1.0);
	const float LOG2 = 1.442695;
	float z = 1.0 - (gl_FragCoord.z / gl_FragCoord.w)/100; // format: 1.0 - (z/w) / view distance (fog starts here);
	float fogFactor = exp2( -density * 
					   density * 
					   z * 
					   z * 
					   LOG2 );
	fogFactor = clamp(fogFactor, 0.0, 1.0);

	//LIGHT
	
	// Note: need to balance the starting brightness of our textures
	
	  vec4 Idif;
	  vec4 Ispec;
	  vec4 Iamb = (gl_FrontMaterial.ambient * gl_LightSource[0].ambient);
	  vec3 N = normalize(n);
	  vec3 L = normalize(lightVec);
	  
	 Idif = gl_FrontMaterial.diffuse * gl_LightSource[0].diffuse * max(dot(N, L),0.0);
	 Ispec = gl_FrontMaterial.specular * gl_LightSource[0].specular * pow(max(dot(N,HalfVector),0.0), gl_FrontMaterial.shininess);
	
	vec4 texel;
	vec4 texD;
	vec3 ct,cf;
	float intensity,at,af;
	
	

	texD = blendedTerrain(texD);
	ct = texD.rgb;
	at = texD.a ;
	
	vec4 shadowCoordinateWdivide = shadowTexCoord / shadowTexCoord.w ;
	// Used to lower moiré pattern and self-shadowing
	shadowCoordinateWdivide.z += 0.0005;
	
	float distanceFromLight = texture2D(shadowMap,shadowCoordinateWdivide.st).z;
	
	 float shadow = 1.0;
	 if (shadowTexCoord.w > 0.0)
	 	shadow = distanceFromLight < shadowCoordinateWdivide.z ? 0.8 : 1.0 ;

	vec4 finalColor =  ( Iamb*texD  + Idif*texD + Ispec*texD )*shadow;//  diffuse + specular;

	gl_FragColor = mix(fogColor, finalColor, fogFactor );
	//gl_FragColor = finalColor;
}

bool  FrameBufferObject::initialize(int width_, int height_, int format)
{
	
	if(!GLEW_EXT_framebuffer_object)
	{
		printf("GL_EXT_framebuffer_object not supported");
		return false;
	}

	if(width_ <= 0 || height_ <= 0)
	{
		printf("Width and height of FBO must be positive");
		return false;
	}

  height = height_;
  width  = width_;

  glGenFramebuffersEXT(1, &frameBufferIndex);
  glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, frameBufferIndex);

  GLuint depth = (format & FBO_DEPTH_16) ? GL_DEPTH_COMPONENT16 :
                 (format & FBO_DEPTH_24) ? GL_DEPTH_COMPONENT24 :
                 (format & FBO_DEPTH_32) ? GL_DEPTH_COMPONENT32 : 0;
  if(depth)
  {
    glGenRenderbuffersEXT(1, &depthBufferIndex);
    glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, depthBufferIndex);
    glRenderbufferStorageEXT(GL_RENDERBUFFER_EXT, depth, width, height);
    glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, 
                                 GL_RENDERBUFFER_EXT, depthBufferIndex);

  }

  if(format & FBO_STENCIL)
  {
    glGenRenderbuffersEXT(1, &stencilBufferIndex);
    glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, stencilBufferIndex);
    glRenderbufferStorageEXT(GL_RENDERBUFFER_EXT, GL_STENCIL_INDEX, width, height);
    glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_STENCIL_ATTACHMENT_EXT,
                                 GL_RENDERBUFFER_EXT, stencilBufferIndex);
  }
  bool result = checkFrameBufferStatus();
  glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0);
  return result;
}

void ShadowMap()
{

	
	//Create the shadow map texture
	glGenTextures(1, &shadowMapTexture);
	glBindTexture(GL_TEXTURE_2D, shadowMapTexture);

	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);

	glTexImage2D( GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, shadowMapSize, shadowMapSize, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE, 0);
	glBindTexture(GL_TEXTURE_2D, 0);

	//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
	//glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_LUMINANCE);
	//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
	fbo = new FrameBufferObject();
	bool working = fbo->initialize(shadowMapSize,shadowMapSize,0);
	fbo->bind();
	// Instruct openGL that we won't bind a color texture with the currently binded FBO
	  glDrawBuffer(GL_NONE);
	  glReadBuffer(GL_NONE);
	fbo->stop();


}

void FirstPass()
{
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	fbo->bind();
	fbo->switchDepthTarget(GL_TEXTURE_2D, shadowMapTexture);
	//First pass - from light's point of view
	
	//Use viewport the same size as the shadow map
	
	glMatrixMode(GL_PROJECTION);
	glLoadMatrixf(lightProjectionMatrix);

	glMatrixMode(GL_MODELVIEW);
	glLoadMatrixf(lightViewMatrix);

	glViewport(0, 0, shadowMapSize, shadowMapSize);

	//Draw back faces into the shadow map
	glCullFace(GL_FRONT);

	//Disable color writes, and use flat shading for speed
	glShadeModel(GL_FLAT);
//	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);

	gFrustum.update();
	//Renders all actors in the scene
	NxU32 nbActors = gScene->getNbActors();
    NxActor** actors = gScene->getActors();
    while (nbActors--)
	{
        NxActor* actor = *actors++;
        DrawActor(actor, gSelectedActor, true);
    }
	terr.RenderALLHeightMap(true);

		const GLfloat bias[16] = {	
			0.5, 0.0, 0.0, 0.0, 
			0.0, 0.5, 0.0, 0.0,
			0.0, 0.0, 0.5, 0.0,
		0.5, 0.5, 0.5, 1.0};

	glMatrixMode(GL_TEXTURE);
	glActiveTextureARB(GL_TEXTURE7);
	glLoadIdentity();	
	glLoadMatrixf(bias);
	glMultMatrixf(lightProjectionMatrix.entries); // now multiply by the matrices we have retrieved before
	glMultMatrixf(lightViewMatrix.entries);
	glMatrixMode(GL_MODELVIEW);

	fbo->stop();
}

void SecondPass()
{
	//2nd pass - Draw from camera's point of view
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	
	//Use dim light to represent shadowed areas

	glActiveTextureARB(GL_TEXTURE7);
	glBindTexture(GL_TEXTURE_2D, shadowMapTexture);
	glEnable(GL_TEXTURE_2D);

	gFrustum.update();
	//Renders all actors in the scene
	NxU32 nbActors = gScene->getNbActors();
    NxActor** actors = gScene->getActors();
	terr.RenderHeightMap(true);
    while (nbActors--)
    {
        NxActor* actor = *actors++;
        DrawActor(actor, gSelectedActor, true);
    }
	
}

Don’t cast front faces into the shadow map. Cast back faces.

If that doesn’t fix it. Stop rendering your terrain into the shadow maps to see if your shadowing results for the boxes look even half-reasonable.

I’m using front face culling; even when i don’t render the terrain into the shadow map, i still end up with the entire terrain appearing to be shadowed.

the problem is probably in fbo itself i’m not sure it’s even being drawn to.

Ok, so I tried setting the shadow texture to GL_RGB and the texture is getting drawn to properly when i render it straight to a quad. just for some reason the when i try using a depth texture is doesn’t work properly

Ok, so the depths you are writing on the light-space pass are somehow wrong, or your depth lookups on the camera-space pass are somehow wrong.

Did you go checking for GL errors?

the problem is probably in fbo itself i’m not sure it’s even being drawn to. … Ok, so I tried setting the shadow texture to GL_RGB and the texture is getting drawn to properly when i render it straight to a quad. just for some reason the when i try using a depth texture is doesn’t work properly

Ok, let’s go with that. So you agree that when you are rendering these shadow maps, they need to be to textures, not renderbuffers, because you want to rebind them to shader samplers (texture units) later to read from, right? Further, to use hardware depth comparisons, they need to be “depth” textures. Right?

Ok, let’s go surfing your code. A bit hard to trace because you just posted a dump with your abstraction schemes in place. In the future, flatten it to pure GL calls. You’ll often find your bug this way.

Anyway, in your FBO init, presumably this “setup depth renderbuffer” stuff isn’t used, because you pass in a format of 0:

  if(depth)
  {
    glGenRenderbuffersEXT(1, &depthBufferIndex);
    glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, depthBufferIndex);
    glRenderbufferStorageEXT(GL_RENDERBUFFER_EXT, depth, width, height);
    glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, 
                                 GL_RENDERBUFFER_EXT, depthBufferIndex);

  }

This’d be wrong anyway, because we want a depth texture bound. So surfing on…

shadow texture setup:

	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);

GL_CLAMP_TO_EDGE is more likely what you want, as CLAMP uses the border texel. And I’m not sure but I don’t think a border texel is even valid for depth textures (though I’ve never searched the spec for that).

	glTexImage2D( GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, shadowMapSize, shadowMapSize, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE, 0);

Your first GL_DEPTH_COMPONENT is fine, though it could also be a specific depth component internal format. E.g. GL_DEPTH_COMPONENT24 being the most common hardware accelerated format. This is the argument that actually determines the format of the depth texture, GL_DEPTH_COMPONENT just giving you some driver/GPU default format.

	//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
	//glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_LUMINANCE);
	//glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);

Why is this commented out? Looks fine to me! You’re gonna need it uncommented for hardware depth comparisons to work in your scene-render pass.

The only comment I’d offer is that GL_DEPTH_TEXTURE_MODE is really only useful pre-GLSL 1.3. With GLSL 1.3+, sampling a depth texture always behaves as if you specified GL_LUMINANCE for this (LUMINANCE = rrr1, where r is the shadow comparison value). Having it here shouldn’t hurt though, and LUMINANCE is a good default to assume.

	bool working = fbo->initialize(shadowMapSize,shadowMapSize,0);

So here you specify that format = 0, which suppresses allocating a depth or stencil renderbuffer for the FBO – it’s basically empty at this point.

Therefore, I’ll bet that checkFrameBufferStatus() call in the FBO setup would have told you this FBO is “not” ready for rendering to. But you’re not checking that status.

render for shadow:

	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	fbo->bind();
	fbo->switchDepthTarget(GL_TEXTURE_2D, shadowMapTexture);
        ... now go draw some stuff ...

You don’t actually tell us how this switchDepthTarget() thing is implemented, so being a skeptic, I’ll assume there are bugs in that.

Also, you’re executing glClear before you actually bound the FBO and setup its depth attachment. So keep in mind you are not clearing the FBO depth texture to the far clip value (1 typically), which could very well indicate why your shadowing lookups are trashed.

Also, “if” you were clearing the depth FBO/texture here, there would be absolutely no reason to be clearing the color buffer too, because there isn’t one!

So at the very least, rip the COLOR_BUFFER_BIT off this glClear, and move the glClear down past the FBO bind and setting the depth attachment to be the depth texture.

Also…

scene render:

glEnable(GL_TEXTURE_2D);

This is meaningless when you’re using shaders.

Thanks for your help, seems the problem was actually where i setup my light matrices ( The function that did that never got called )
And partly due to my misunderstanding of fbo’s.

Now the error seems to be in the shaders themselves because the depth texture is being written properly (rendering it to a poly shows a correct view from the light of the objects and their depths) however the shader it rendering the entire terrain black.

I’m sure i’ll figure out why this it isn’t work, guess this goes to show you shouldn’t just copy some code and hope it works without understanding it first

Thank you very much for your help.

Is it possible the issue is that i’m trying to put a shadow map of size (3360x2100) onto a terrain that is 2048x2048?

Why would that be a problem?

You’re using 2D textures (not RECT textures) so texcoords should be normalized 0…1, isolating your shader from the shadow map res.

That said, anything is possible with bugs.