transparent shadow texture

Hello, I’m using shadowmapping to cast a shadow from one mesh(m1) onto another mesh(m2). However, there’s a little problem with the color channels. I don’t know how to use them:

The idea is that I render ‘m1’ to a shadowmap texture and use that texture to draw black opaque pixels where the shadow is and fully transparant pixels where there’s no shadow. I already solved the problem of going from texture space to camera space. The only problem is I don’t know how I can tell openGL to handle the texture channels the way I want it to.

The texture is set up like this(in C++):

[b]

    glGenTextures(1, &shadowmap);
    glBindTexture(GL_TEXTURE_2D, shadowmap);
    glTexImage2D( GL_TEXTURE_2D, 0, GL_RGB,512,512, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,        GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,        GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,            GL_CLAMP);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,            GL_CLAMP);

 [/b]
[/b][/QUOTE]

Right now, I'm using the following procedure to make a shadow map:


[b]

glDisable(GL_LIGHTING);
glClearColor(0,0,0,0);
glCullFace(GL_FRONT);
glShadeModel(GL_FLAT);

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    glMatrixMode(GL_PROJECTION);
    glLoadMatrixf(lightProjectionMatrix.m);

    glMatrixMode(GL_MODELVIEW);
    glLoadMatrixf(lightViewMatrix.m);

    //Use viewport the same size as the shadow map
glPushAttrib(GL_VIEWPORT_BIT);
    glViewport(0, 0, 512, 512);

glBindTexture(GL_TEXTURE_2D,0);
glColor4f(1,1,1,1);

///////////////////////////////
///This is where I draw 'm1'///
///////////////////////////////

glEnable(GL_LIGHTING);
glBindTexture(GL_TEXTURE_2D, shadowmap);

// Write the framebuffer to texture //
glCopyTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, 0, 0, 512, 512,0);

   glPopAttrib();
glCullFace(GL_BACK);
glShadeModel(GL_SMOOTH);
glEnable(GL_LIGHTING);

[/b]
[/b][/QUOTE]

And this is what I do when rendering the scene:

[b]
    glClearColor(0.0f, 0.0f, 0.3f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);

    glMatrixMode(GL_PROJECTION);
    glLoadMatrixf(cameraProjectionMatrix.m);

    glMatrixMode(GL_MODELVIEW);
    glLoadMatrixf(cameraViewMatrix.m);

    glLightfv(GL_LIGHT0,GL_POSITION,light_pos);
    glEnable(GL_LIGHTING);
    glBindTexture(GL_TEXTURE_2D,tex_m2);


    ///////////////////////////////
    ///This is where I draw 'm2'///
    ///////////////////////////////

    matrix4 tm=mat_bias*lightProjectionMatrix*lightViewMatrix;
    GLfloat row[4];

    row[0]=tm.m11; row[1]=tm.m12; row[2]=tm.m13; row[3]=tm.m14;
       glTexGeni(GL_S, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
    glTexGenfv(GL_S, GL_EYE_PLANE, row);
    glEnable(GL_TEXTURE_GEN_S);

    row[0]=tm.m21; row[1]=tm.m22; row[2]=tm.m23; row[3]=tm.m24;
        glTexGeni(GL_T, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
        glTexGenfv(GL_T, GL_EYE_PLANE, row);
    glEnable(GL_TEXTURE_GEN_T);

    row[0]=tm.m31; row[1]=tm.m32; row[2]=tm.m33; row[3]=tm.m34;
        glTexGeni(GL_R, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
        glTexGenfv(GL_R, GL_EYE_PLANE, row);
    glEnable(GL_TEXTURE_GEN_R);

    row[0]=tm.m41; row[1]=tm.m42; row[2]=tm.m43; row[3]=tm.m44;
        glTexGeni(GL_Q, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
        glTexGenfv(GL_Q, GL_EYE_PLANE, row);
    glEnable(GL_TEXTURE_GEN_Q);

    glBindTexture(GL_TEXTURE_2D,shadowmap);
    glColor4f(0,0,0,1);


    ////////////////////////////////////////////////////
    ///This is where I draw 'm2'again for the shadow ///
    ////////////////////////////////////////////////////

        glDisable(GL_TEXTURE_GEN_S);
        glDisable(GL_TEXTURE_GEN_T);
        glDisable(GL_TEXTURE_GEN_R);
        glDisable(GL_TEXTURE_GEN_Q);

    glBindTexture(GL_TEXTURE_2D,tex_m1);

    ///////////////////////////////
    ///This is where I draw 'm1'///
    ///////////////////////////////
 [/b]
[/b][/QUOTE]

With this code, I see an opaque black 'm2' with a white shadow of 'm1' cast upon it. m1 is shown in the scene, colored as it should be.
I have tried to set all the GL_RGBs in the texture generation to GL_ALPHAs. This just results in a totally white m2 and no shadow.

Like I said earlier, that's not what I want. I just want a shadowmap that's drawn black where the shadow is and invisble where no shadow is.

Could someone explain to me how I can manipulate the coloring?

Have you tried using the alpha channel, alpha funcs and blending ? Usually you should draw your shadow with grey or black color with an alpha channel about 0.5, using good blending functions.
So something like this:


DrawStuff();
glEnable (GL_BLENDING);
glBlendFunc(GL_SOURCE_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glColor4f (.3,.3,.3,.7); // .7 means almost opaque
DrawShadowProjection();

I changed the shadowmap texture’s color formats to GL_RGBA and changed the shadow drawing procedure to this:


	glBindTexture(GL_TEXTURE_2D,shadowmap);
	glEnable(GL_BLEND);
	glDepthMask(GL_FALSE);
	glBlendFunc(GL_SRC_ALPHA,GL_ONE_MINUS_SRC_ALPHA);
	glColor4f (.3,.3,.3,.5);

	///////////////////////////
	/////// draw m2 ///////////
	///////////////////////////

	glDepthMask(GL_TRUE);
	glDisable(GL_BLEND);

	glColor4f (1,1,1,1);

Still an opaque black ‘m2’ is rendered. Weird though, 'cos right before m1 is drawn to the shadowmap, the buffer is cleared to 0,0,0,0. I know my m1 is drawn with color white to the shadowmap. But that should at least get me a white transparent shadow, shouldn’t it?

OK news, I just learned that glCopyTexImage2D can’t render alpha channels to textures in openGL on windows. So I need to find a way around it.

Guess I need to render black shadow on white background and not render the white while drawing the shadow.

Is there any way I could make openGL render all but a specific color( the white )?

yes , just use that color when rendering the shadow.
But you do need to start from a black background and then draw only what the light source can reach.

So basically you need to render a white non shadow on black