PDA

View Full Version : Shadow map problem using GLSL with Catalyst 10.7



shiyujia
08-14-2010, 03:34 AM
I wrote a shadow map demo program using GLSL. It used to work with Catalyst driver 9.12 on HD 4830. And it also works on a NVidia Geforce 8800 GTS 320 with 258.96 driver. Now after installing Catalyst 10.7 on the HD 4830, the program does not work properly. It seems shadows are either clipped or there are no shadows at all. Here are the GLSL source codes:
Vertex shader:


uniform mat4 Model; // Model matrix
uniform mat4 ModelIT; // Inverse transpose of Model matrix
uniform mat4 ModelViewProj; // Model View Projection matrix

uniform vec2 texcoord_scaling; // Texture coordinate scaling factors
uniform vec2 texcoord_bias; // Texture coordinate bias

uniform vec3 viewer_position;

varying vec2 diffuse_map_texcoord;
varying vec3 position_in_world_space;
varying vec3 normal_in_world_space;
varying vec3 dir_to_viewer;

void main()
{
gl_Position=ModelViewProj*gl_Vertex;
diffuse_map_texcoord=gl_MultiTexCoord0.xy*texcoord _scaling+texcoord_bias;
position_in_world_space=(Model*gl_Vertex).xyz;
normal_in_world_space=(ModelIT*vec4(gl_Normal, 0.0)).xyz;

dir_to_viewer=normalize(viewer_position-position_in_world_space);
}

Fragment shader:


#define NUM_LIGHTS 3
#define NUM_LIGHTS_WITH_SHADOW 2
#define NUM_SHADOWMAP_SAMPLES 2

struct CLight
{
bvec2 enabled_flags; // x--Light enabled, y--Shadow enabled

vec4 pos; // Light position
vec3 dir; // Light Direction
vec4 color; // Light Color
vec3 afactors; // Attenuation factors
vec2 misc; // x--Cosine of cut off angle, y--Angular falloff factor
};

vec4 Illuminate(CLight lt, vec3 P, out vec3 L)
// Input:
// lt: light source
// P: Position of the point to be illuminated
// Output:
// L: Direction to light unit vector
// Return value: Light intensity at point P
{
vec3 to_light=lt.pos.xyz-P*lt.pos.w;
L=normalize(to_light);

float d=length(to_light); // Distance to light
float fd=1.0/(lt.afactors.x+d*(lt.afactors.y+d*lt.afactors.z)); // Radial attenuation

float cosine_theta=dot(lt.dir, -L);
float fa=(cosine_theta-lt.misc.x)/(1.0-lt.misc.x);
fa=pow(clamp(fa, 0.0, 1.0), lt.misc.y); // Angular attenuation

return lt.color*fa*fd;
}

// Shadow map sample points in pairs (xy and zw)
vec4 shadow_map_sample_points[]=vec4[NUM_SHADOWMAP_SAMPLES](
vec4(-0.4, -0.2, 0.2, -0.4),
vec4(0.4, 0.2, -0.2, 0.4)
);

vec4 ShadowMapFiltering(sampler2DShadow shadow_map, vec4 texcoord, vec2 kernel_size)
{
vec4 shadow_mask=vec4(0.0);
vec2 dtexcoord=texcoord.w*kernel_size;

for (int k=0; k<NUM_SHADOWMAP_SAMPLES; ++k)
{
vec4 tap, shadow_sample;
tap=texcoord+vec4(shadow_map_sample_points[k].xy*dtexcoord, 0.0, 0.0);
shadow_sample=shadow2DProj(shadow_map, tap);
shadow_mask+=shadow_sample;
tap=texcoord+vec4(shadow_map_sample_points[k].zw*dtexcoord, 0.0, 0.0);
shadow_sample=shadow2DProj(shadow_map, tap);
shadow_mask+=shadow_sample;
}

shadow_mask/=vec4(2.0*float(NUM_SHADOWMAP_SAMPLES));

return shadow_mask;
}

uniform CLight lights[NUM_LIGHTS]; // Lights
uniform sampler2DShadow shadow_maps[NUM_LIGHTS_WITH_SHADOW]; // Shadow maps
uniform mat4 shadow_map_matrices[NUM_LIGHTS_WITH_SHADOW]; // Projective matrices for shadow maps

uniform bvec2 texture_flags;
// x--Diffuse map enabled, y--Specular map enabled

uniform sampler2D diffuse_map;
uniform sampler2D specular_map;

uniform vec4 global_ambient;

uniform vec4 diffuse;
uniform vec4 specular;
uniform float shininess;

varying vec2 diffuse_map_texcoord;
varying vec3 position_in_world_space;
varying vec3 normal_in_world_space;
varying vec3 dir_to_viewer;

void main()
{
int k;

// Get base diffuse and specular color
vec4 tex_color;
vec4 base_diffuse_color=diffuse;
vec4 base_specular_color=specular;

if (texture_flags.x)
{
tex_color=texture2D(diffuse_map, diffuse_map_texcoord);
base_diffuse_color*=tex_color;
}

if (texture_flags.y)
{
tex_color=texture2D(specular_map, diffuse_map_texcoord);
base_specular_color*=tex_color;
}

// Calculate vectors
vec3 N=normalize(normal_in_world_space);
vec3 V=normalize(dir_to_viewer);
vec3 L; // Direction to light vector
vec3 H; // Halfway vector

// Global ambient contribution
vec4 diffuse_color=global_ambient;
vec4 specular_color=vec4(0.0);

// Contribution from lights with shadows
vec4 lcolor;
for (k=0; k<NUM_LIGHTS_WITH_SHADOW; ++k)
if (lights[k].enabled_flags.x)
{
lcolor=Illuminate(lights[k], position_in_world_space, L);

if (lights[k].enabled_flags.y)
{
vec4 shadow_map_texcoord=shadow_map_matrices[k]*vec4(position_in_world_space, 1.0);
vec4 shadow_mask=ShadowMapFiltering(shadow_maps[k],
shadow_map_texcoord, vec2(0.001, 0.001));
lcolor*=shadow_mask;
}

H=normalize(L+V);
float diffuse_factor=max(dot(N,L), 0.0);
float specular_factor=pow(max(dot(N,H), 0.0), shininess);

diffuse_color+=lcolor*diffuse_factor;
specular_color+=lcolor*specular_factor;
}

// Contribution from lights without shadows
for (k=NUM_LIGHTS_WITH_SHADOW; k<NUM_LIGHTS; ++k)
if (lights[k].enabled_flags.x)
{
lcolor=Illuminate(lights[k], position_in_world_space, L);

H=normalize(L+V);
float diffuse_factor=max(dot(N,L), 0.0);
float specular_factor=pow(max(dot(N,H), 0.0), shininess);

diffuse_color+=lcolor*diffuse_factor;
specular_color+=lcolor*specular_factor;
}

diffuse_color*=base_diffuse_color;
specular_color*=base_specular_color;

gl_FragColor=diffuse_color+specular_color;
gl_FragColor.a=base_diffuse_color.a;
}

If I comment out these two lines:


tex_color=texture2D(specular_map, diffuse_map_texcoord);
base_specular_color*=tex_color;

then it works. I think there is something weird going on with 10.7's GLSL compiler.

DmitryM
08-14-2010, 06:53 AM
... or your specular_map and/or diffuse_map_texcoord is not properly defined for GL. Why don't you just output the specular map for a test?

Dark Photon
08-14-2010, 08:45 AM
I wrote a shadow map demo program using GLSL. It used to work with Catalyst driver 9.12 on HD 4830. And it also works on a NVidia Geforce 8800 GTS 320 with 258.96 driver. Now after installing Catalyst 10.7 on the HD 4830, the program does not work properly. It seems shadows are either clipped or there are no shadows at all. ... If I comment out these two lines ... then it works. I think there is something weird going on with 10.7's GLSL compiler.
DmitryM's got you covered.

While ATI's GL/GLSL compiler in some cases (in my experience) isn't quite as aggressive at optimizing or robust as NVidia's, in other cases it is more strict. So it is worth tracking the cause down because it might just as well be yours or ATI's.

One thing. You need:

#version 120

for your use of arrays. Also while you probably are, be sure you are querying your COMPILE_STATUS after compile, your LINK_STATUS after link, and your VALIDATE_STATUS after validate (the latter right before you use the program, for debugging purposes). In my experience, I've never gotten anything from NVidia for ValidateProgram, but with ATI I have. You'll catch things like the wrong type of texture bound to a texture unit and such.

shiyujia
08-16-2010, 02:40 AM
My demo program already has codes to check compilation and linking status of a shader program. And the shader program is compiled and linked successfully. However it did not have codes to validate program. So I add validation to it and the shader program indeed fails validation. However I do not understand how this error is possible. The validation error is:
"Different sampler types for same sample texture unit in fragment shader."
I saw another thread in this forum discussing the same validation error issue on ATI Radeon HD cards. However the issue is not resolved.

DmitryM
08-16-2010, 04:15 AM
I guess that means you have 2+ samplers of different types (sampler2D vs sampler2DShadow) bound to the same texture unit ID (via glUniform1i call). There might be a logical error somewhere in your shader initialization/binding routines.

Dark Photon
08-16-2010, 05:17 AM
"Different sampler types for same sample texture unit in fragment shader."
I think it may mean that at the time you call ValidateProgram, you have something like this active:



glActiveTexture( GL_TEXTURE0 );
glBindTexture( GL_TEXTURE_2D, 3 ); // NOTE: 2D texture


But in your shader code you have something like this:


sampler2DArray my_tex; // NOTE: 2D array texture

and let's say you've stored a 0 on the my_tex uniform with glUniform1i to state that you want it to sample texture unit 0.

In other words, go through each of your samplers/texture units and verify that the texture target you are passing glBindTexture matches the type of the sampler in the shader.

Also before you even do this, be very sure that you are not calling glValidateProgram() until you have the GL pipeline state set up (e.g. textures bound to texture units, uniforms set up, etc.) ready to render with that shader.

shiyujia
08-16-2010, 05:47 AM
Thanks for all your replies. Now I put validation codes after all uniforms variables are set and the shader program is validated successfully.
I did some experimenting with the shader source code and found the following:
If I put the following code at the end of the fragment shader main() function:


gl_FragColor=base_diffuse_color;

or


gl_FragColor=base_specular_color;

Then the "Different sampler types for same sample texture unit in fragment shader." validation error returns again, even after all uniforms variables are set.

DmitryM
08-16-2010, 06:19 AM
In this case the shader compiler may skip the whole loop, making the "shadow_maps[]" samplers unused. Then you query GL for active uniforms and don't get these samplers, skipping their binding on your side, what in result cases this validation notice.

The conclusion is: don't pay attention to it (in the artificial case). Go back to your original problem and show us the screen shots of:
-base diffuse only
-base specular only
-diffuse
-specular
-combined

shiyujia
08-17-2010, 02:51 AM
I made a small mistake. The previous driver I used on my HD4830 is Catalyst 9.4, not 9.12.
To minimize sources of error, I enabled only one light source in the program.
Here is a screen capture of the program running under 9.4:
http://imgur.com/L7yjR.jpg
And here is what it looks like under 10.7:
http://imgur.com/KMrP0.jpg
Here is the shadow map itself, generated using frame buffer object under 10.7:
http://imgur.com/XFgQG.jpg
Replacing the entire body of fragment shader main() function with


gl_FragColor=diffuse*texture2D(diffuse_map, diffuse_map_texcoord);

gives this result:
http://imgur.com/bN9GR.jpg
And replacing the entire body of fragment shader main() function with


gl_FragColor=specular*texture2D(specular_map, diffuse_map_texcoord);

gives this result:
http://imgur.com/Z1p7t.jpg
I haven't been able to find anything wrong in my code yet.

DmitryM
08-17-2010, 10:38 AM
Simplify the test case:
-make it be affected only by one light
-remove shadow texture filtering
-remove loops, uniform arrays, branches

From what I see, you either pass the shadow map incorrectly or sample from it with wrong coordinates. Try outputting the shadow texture coordinate into the color.

shiyujia
08-18-2010, 02:54 AM
OK, I finally found out what is wrong. And I am quite certain it is a driver bug in Catalyst 10.7.
I was tracing the program in debug mode and found out that glGetUniformLocation() returned -1 for all the shadow map sampler variables. It seems that the GLSL compiler in Catalyst 10.7 does not recognize that the shadow maps are being used. So I changed the following code in the fragment shader:


vec4 shadow_mask=ShadowMapFiltering(shadow_maps[i], shadow_map_texcoord, vec2(0.001, 0.001));

to


vec4 shadow_mask=shadow2DProj(shadow_maps[i], shadow_map_texcoord);

And everything is rendered correctly now!
So it seems that the GLSL compiler in Catalyst 10.7 fails to consider arguments of a function as being used.

shiyujia
08-21-2010, 02:35 AM
I got confirmation of this driver bug from an AMD employee, and I was told that it will be fixed in the next Catalyst driver release.

shiyujia
08-26-2010, 10:00 PM
I just installed the newly released 10.8 Catalyst driver and the bug is indeed fixed.