Raycast selection/hit -- About 90% there with one small glitch.

Moderators… If you are reading this, could you please delete this older post of mine? I am asking another question that is similar, but worded better with better examples of my problem and I don’t want to confuse anyone who may be able to help. Thanks…

I have spent all day trying to figure this out and I feel like I am about 90% there. I have made a screencast video of my problem. It is in MP4 format. If you are interested, you can download/view it by clicking here.

ETA: here is another video of the object rotating. It shows more of the problem.

I am using this Matrix and Vector classes found here.

This is sloppy, sloppy code, I know – But, I am just trying to get this to work before I start organizing it so I know how to organize it.

Please watch the video above to better understand what I am trying to do…

void CPCSView::OnLButtonDown(UINT nFlags, CPoint point){


    Vector3 vecRayNear,vecRayFar;
    { // find the ray based on click point
        //GLdouble dblMatMV[16];
        GLdouble dlbMatPM[16];
        GLint nArrViewPort[4];
        
        //glGetDoublev(GL_MODELVIEW_MATRIX,dblMatMV); // don't need this, it's stored in m_matModelView
        glGetDoublev(GL_PROJECTION_MATRIX, dlbMatPM);
        glGetIntegerv(GL_VIEWPORT, nArrViewPort);
        
        double dblArrNear[3];
        gluUnProject(point.x, point.y, 0.0, m_matModelView.getTransposed(), dlbMatPM,
                     nArrViewPort, &dblArrNear[0], &dblArrNear[1],&dblArrNear[2]);
        
        vecRayNear = Vector3((float)dblArrNear[0],(float)dblArrNear[1],(float)dblArrNear[2]);


        double dblArrFar[3];
        gluUnProject(point.x, point.y, 1.0, m_matModelView.getTransposed(), dlbMatPM,
                     nArrViewPort, &dblArrFar[0], &dblArrFar[1],&dblArrFar[2]);


        vecRayFar = Vector3((float)dblArrFar[0],(float)dblArrFar[1],(float)dblArrFar[2]);
    }    


    // calculate the triangle points and normal
    Vector3 vecTri[3];
    Vector3 vecNormal;
    {
        // original triangle
        //glVertex3f(1, 0, 0);
        //glVertex3f(0, 1, 0);
        //glVertex3f(0, 0, 2);
        const Matrix4 mvT(m_matModelView);
        vecTri[0] = Vector3(1,0,0)*mvT;
        vecTri[1] = Vector3(0,1,0)*mvT;
        vecTri[2] = Vector3(0,0,2)*mvT;
        vecNormal = (vecTri[1]-vecTri[0]).cross(vecTri[2]-vecTri[0]);
        vecNormal.normalize();
    }


    { // this is normal "where does a line (in this case the ray) intersect a triangle" code
        const float flDist1 = (vecRayNear-vecTri[0]).dot(vecNormal);
        const float flDist2 = (vecRayFar-vecTri[0]).dot(vecNormal);
        if((flDist1 * flDist2) >= 0.0f){
            TRACE0("line doesn't cross the triangle.
");
        }
        if(flDist1 == flDist2){
            TRACE0("line and plane are parallel.
");
        }


        const Vector3 vecIntersect  = vecRayNear+(vecRayFar-vecRayNear)*(-flDist1/(flDist2-flDist1));


        bool bFoundIntersection = true;
        Vector3 vecTest;


        vecTest = vecNormal.cross(vecTri[1]-vecTri[0]);
        if(vecTest.dot(vecIntersect-vecTri[0])<0.0f){
            TRACE0("No intersection 1
");
            bFoundIntersection = false;
        }


        vecTest = vecNormal.cross(vecTri[2]-vecTri[1]);
        if(vecTest.dot(vecIntersect-vecTri[1])<0.0f){
            TRACE0("No intersection 2
");
            bFoundIntersection = false;
        }


        vecTest = vecNormal.cross(vecTri[0]-vecTri[2]);
        if(vecTest.dot(vecIntersect-vecTri[0])<0.0f){
            TRACE0("No intersection 3
");
            bFoundIntersection = false;
        }


        if(bFoundIntersection){
            m_vecClicks.push_back(vecIntersect);
            TRACE3("Found Hit: %g; %g; %g
",vecIntersect.x,vecIntersect.y,vecIntersect.z);
            Repaint();
        }
    }
    CGLEnabledView::OnLButtonDown(nFlags, point);
}

Any insight would be greatly appreciated!

What is your question? You need insights about what?

The question is in the video I made.

I will type out the question/problem when I get home, it was just easier to make a video than try to explain with text.

OK… i found the problem. First off, i shouldn’t be using the model view transform to alter the triangle before testing if the ray intersects it, secondly, gluUnProject’s 2nd parameter shouldn’t simply be point.y, since OGL’s Y axis is inverted, i had to change it to nArrViewPort[3]-point.y. That’s what was causing it to invert.