Here you go. Worked on the first try.
The code below puts a light in the center of the frustum and sets a radius of the half frustum size to make the expected depth bounds 0.0 and 1.0 and that just works.
I added some rotations then and except for floating point rounding errors that gets the same result.
Hmm, I didn’t test rotations which turn more than 90 degrees, but I’m confident.
I guess the “divide by w” step is the key to your inverted results. Watch your matrices.
Have fun.
#define EPSILON 0.00001
GLdouble matProjection[16];
GLdouble matModelview[16];
GLint viewport[4];
GLdouble dir[3]; // modelspace direction
GLdouble norm;
GLdouble posNear[3];
GLdouble posFar[3];
GLdouble xWin, yWin;
GLdouble zLight, zNear, zFar;
int i;
GLdouble lightPos[3] = {0.0, 0.0, 0.0};
GLdouble lightRadius = 4.0; // Means light borders are exactly on the zNear and zFar frustum planes.
// Expected result is zNear == 0.0 and zFar == 1.0
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
gluPerspective(10.0, 1.0, 1.0, 9.0);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
// Center the modelspace origin in viewing frustum.
glTranslatef(0.0f, 0.0f, -5.0f);
// Rotate a little to make it more "real world-ish".
glRotatef(30.0f, 1.0f, 0.0f, 0.0f);
glRotatef(45.0f, 0.0f, 1.0f, 0.0f);
glGetDoublev(GL_PROJECTION_MATRIX, matProjection);
glGetDoublev(GL_MODELVIEW_MATRIX, matModelview);
glGetIntegerv(GL_VIEWPORT, viewport);
// Eye-space is right-handed, viewing direction is (0, 0, -,1, 0)
// Transforming this direction back into modelspace requires the inverse modelview matrix.
// For vectors, translation is irrelevant, and if there is no scaling,
// the inverse is the transpose (means only rotation affects the vector)
// Means, if the modelview matrix in OpenGL float[16] column major is depicted as
// a_0, a_4, a_8, a12
// a_1, a_5, a_9, a13
// a_2, a_6, a10, a14
// a_3, a_7, a11, a15
// The inverse is
// a_0, a_1, a_2, a_3
// a_4, a_5, a_6, a_7
// a_8, a_9, a10, a11
// a12, a13, a14, a15
// Multiplied by the eye-space viewing direction
// a_0, a_1, a_2, a_3 0 -a_2
// a_4, a_5, a_6, a_7 * 0 = -a_6
// a_8, a_9, a10, a11 -1 -a10
// a12, a13, a14, a15 0 -a14
// we only need the direction, so if w == -a14 == 0.0 we're set.
// Otherwise divide by w (this can flip the direction if w is negative, and it is!)
dir[0] = -matModelview[ 2];
dir[1] = -matModelview[ 6];
dir[2] = -matModelview[10];
if (fabs(matModelview[14]) > EPSILON)
{
dir[0] /= -matModelview[14];
dir[1] /= -matModelview[14];
dir[2] /= -matModelview[14];
}
// Normalize the modelspace direction. This shouldn't be zero.
norm = sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2]);
// If the light in modelspace is at lightPos
// the worldspace projection of posNear and posFar are the nearest and farthest points
// We need the projected z-values in the end.
for (i = 0; i < 3; i++)
{
dir[i] /= norm; // Normalize
dir[i] *= lightRadius; // Scale with light radius in modelspace.
posNear[i] = lightPos[i] - dir[i];
posFar[i] = lightPos[i] + dir[i];
}
gluProject(lightPos[0], lightPos[1], lightPos[2],
matModelview, matProjection, viewport,
&xWin, &yWin, &zLight); // For verification.
// Interestingly a point in the center of the eye-space z-range gets mapped to 0.9 with the above setup.
// That shows how different the depth value distribution is between near and far values.
gluProject(posNear[0], posNear[1], posNear[2],
matModelview, matProjection, viewport,
&xWin, &yWin, &zNear);
gluProject(posFar[0], posFar[1], posFar[2],
matModelview, matProjection, viewport,
&xWin, &yWin, &zFar);