Trackball issue with Raypicking

[Gorghino]

I'm trying to develop a 3D raypicking in my OpenGL scene.

I have a working OBJ loader with a trackball.

char*      model_file = NULL;       /* name of the obect file */
GLuint     model_list = 0;      /* display list for object */
GLMmodel*  model;               /* glm model data structure */
GLfloat    scale;               /* original scale factor */
GLfloat    smoothing_angle = 90.0;  /* smoothing angle */
GLfloat    weld_distance = 0.00001; /* epsilon for welding vertices */
GLboolean  facet_normal = GL_FALSE; /* draw with facet normal? */
GLboolean  bounding_box = GL_FALSE; /* bounding box on? */
GLboolean  spheres = GL_FALSE;
GLboolean  performance = GL_FALSE;  /* performance counter on? */
GLboolean  stats = GL_FALSE;        /* statistics on? */
GLuint     material_mode = 0;       /* 0=none, 1=color, 2=material, 3=texture */
GLint      entries = 0;         /* entries in model menu */
GLdouble   pan_x = 0.0;
GLdouble   pan_y = 0.0;
GLdouble   pan_z = 0.0;
char texnames[1][64] = {"foto_rgb.ppm"};
//char texnames[1][64] = {"grid.ppm"};
GLint w,h;
GLubyte* texture;
 
 
 
void line (void) {
 
    glLineWidth(10);
    //glPointSize(50.2);
    glColor3f(0.0f, 1.0f, 0.0f);      
    glBegin(GL_LINES);
        glVertex3f( m_start.x, m_start.y, m_start.z );
        glVertex3f( m_end.x, m_end.y, m_end.z );
    glEnd();
    glColor3f(1.0f,1.0f,1.0f);
    glLineWidth(1);
 
}
 
 
void lists(void){
    GLfloat ambient[] = { 0.2, 0.2, 0.2, 1.0 };
    GLfloat diffuse[] = { 0.8, 0.8, 0.8, 1.0 };
    GLfloat specular[] = { 0.0, 0.0, 0.0, 1.0 };
    GLfloat shininess = 65.0;
 
    glMaterialfv(GL_FRONT, GL_AMBIENT, ambient);
    glMaterialfv(GL_FRONT, GL_DIFFUSE, diffuse);
    glMaterialfv(GL_FRONT, GL_SPECULAR, specular);
    glMaterialf(GL_FRONT, GL_SHININESS, shininess);
 
    if (model_list)
        glDeleteLists(model_list, 1);
 
    glDisable(GL_TEXTURE_2D);
    /* generate a list */
    switch (material_mode)
    {
     case 0:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT);
        else
            model_list = glmList(model, GLM_SMOOTH);
     break;
     case 1:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT | GLM_COLOR);
        else
            model_list = glmList(model, GLM_SMOOTH | GLM_COLOR);
     break;
     case 2:
        if (facet_normal)
            model_list = glmList(model, GLM_FLAT | GLM_MATERIAL);
        else
            model_list = glmList(model, GLM_SMOOTH | GLM_MATERIAL);
     break;
     case 3:
        glEnable(GL_TEXTURE_2D);
        model_list = glmList(model, GLM_TEXTURE);
//        glDisable(GL_TEXTURE_2D);
     break;
    }
}
 
void init(void){
    gltbInit(GLUT_LEFT_BUTTON);
 
    /* read in the model */
    model = glmReadOBJ(model_file);
    scale = glmUnitize(model);
    glmFacetNormals(model);
    glmVertexNormals(model, smoothing_angle);
 
    if (model->nummaterials > 0)
        material_mode = 2;
 
    /* create new display lists */
    lists();
 
    glEnable(GL_LIGHTING);
    glEnable(GL_LIGHT0);
    glLightModeli(GL_LIGHT_MODEL_TWO_SIDE, GL_TRUE);
    glEnable(GL_DEPTH_TEST);
}
 
void reshape(int width, int height){
    gltbReshape(width, height);
 
    glViewport(0, 0, width, height);
 
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPerspective(60.0, (GLfloat)height / (GLfloat)width, 1.0, 128.0);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glTranslatef(0.0, 0.0, -3.0);
}
 
void display(void){
    static char s[256], t[32];
    static char* p;
    static int frames = 0;
    int i=0,j=0;
 
    glClearColor(1.0, 1.0, 1.0, 1.0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
    glPushMatrix();
 
    glTranslatef(pan_x, pan_y, 0.0);
 
    gltbMatrix();
 
    if(mouseClicked){
        line();
    }
 
    glCallList(model_list);
 
    glPopMatrix();
 
    glutSwapBuffers();
    glEnable(GL_LIGHTING);
}
 
static GLint      mouse_state;
static GLint      mouse_button;
 
void mouse(int button, int state, int x, int y){
    GLdouble model_project[4*4];
    GLdouble proj[4*4];
    GLint view[4];
 
    /* fix for two-button mice -- left mouse + shift = middle mouse */
    if (button == GLUT_LEFT_BUTTON && glutGetModifiers() & GLUT_ACTIVE_SHIFT)
        button = GLUT_MIDDLE_BUTTON;
 
    gltbMouse(button, state, x, y);
 
    mouse_state = state;
    mouse_button = button;
 
    if (state == GLUT_DOWN && button == GLUT_MIDDLE_BUTTON) {
        glGetDoublev(GL_MODELVIEW_MATRIX, model_project);
        glGetDoublev(GL_PROJECTION_MATRIX, proj);
        glGetIntegerv(GL_VIEWPORT, view);
 
        gluProject((GLdouble)x, (GLdouble)y, 0.0,
            model_project, proj, view,
            &pan_x, &pan_y, &pan_z);
        gluUnProject((GLdouble)x, (GLdouble)y, pan_z,
            model_project, proj, view,
            &pan_x, &pan_y, &pan_z);
        pan_y = -pan_y;
    }
 
    if(button == GLUT_LEFT_BUTTON && state == GLUT_DOWN && glutGetModifiers() & GLUT_ACTIVE_CTRL) {
        convert2dto3D(x,y);
    }
    else if(button == GLUT_WHEEL_UP){ // Wheel up
        glmScale(model, 1.25);
        lists();
    }
    else if(button == GLUT_WHEEL_DOWN){ // Wheel down
        glmScale(model, 0.8);
        lists();
    }
 
    glutPostRedisplay();
}
 
void motion(int x, int y){
    GLdouble model[4*4];
    GLdouble proj[4*4];
    GLint view[4];
 
    gltbMotion(x, y);
 
 
    glutPostRedisplay();
}
 
int main(int argc, char** argv){
    int buffering = GLUT_DOUBLE;
    struct dirent* direntp;
    DIR* dirp;
    int models;
 
    glutInitWindowSize(512, 512);
    glutInit(&argc, argv);
 
    while (--argc) {
        if (strcmp(argv[argc], "-sb") == 0)
            buffering = GLUT_SINGLE;
        else
            model_file = argv[argc];
    }
 
    if (!model_file) {
//        model_file = "data/dolphins.obj";
        model_file = "data/boeing_2.obj";
    }
 
    glutInitDisplayMode(GLUT_RGB | GLUT_DEPTH | buffering);
    glutCreateWindow("Smooth");
 
    glutReshapeFunc(reshape);
    glutDisplayFunc(display);
    glutKeyboardFunc(keyboard);
    glutMouseFunc(mouse);
    glutMotionFunc(motion);
 
/* Image data packed tightly. */
    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
    textures();
 
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL);
//    glEnable(GL_TEXTURE_2D);
 
    models = glutCreateMenu(menu);
    dirp = opendir(DATA_DIR);
    if (!dirp) {
        fprintf(stderr, "%s: can't open data directory.\n", argv[0]);
    } else {
        while ((direntp = readdir(dirp)) != NULL) {
            if (strstr(direntp->d_name, ".obj")) {
                entries++;
                glutAddMenuEntry(direntp->d_name, -entries);
            }
        }
        closedir(dirp);
    }
 
 
    init();
 
    glutMainLoop();
    return 0;
}

void convert2dto3D(int xMouse, int YMouse){
    double matModelView[16], matProjection[16]; 
    int viewport[4]; 
 
    glGetDoublev( GL_MODELVIEW_MATRIX, matModelView ); 
    glGetDoublev( GL_PROJECTION_MATRIX, matProjection ); 
    glGetIntegerv( GL_VIEWPORT, viewport );
 
    double winX = (double)xMouse; 
    double winY = viewport[3] - (double)YMouse; 
    gluUnProject(winX, winY, 0.0, matModelView, matProjection, viewport, &m_start.x, &m_start.y, &m_start.z); 
    gluUnProject(winX, winY, 1.0, matModelView, matProjection,  viewport, &m_end.x, &m_end.y, &m_end.z);
    mouseClicked = GL_TRUE;
 
}

Here trackball's functions:

/*
     *  Simple trackball-like motion adapted (ripped off) from projtex.c
     *  (written by David Yu and David Blythe).  See the SIGGRAPH '96
     *  Advanced OpenGL course notes.
     */
 
 
    #include <math.h>
    #include <stdio.h>
    #include <assert.h>
    #include <GL/glut.h>
    #include "gltb.h"
 
 
    #define GLTB_TIME_EPSILON  10
 
 
    static GLuint    gltb_lasttime;
    static GLfloat   gltb_lastposition[3];
 
    static GLfloat   gltb_angle = 0.0;
    static GLfloat   gltb_axis[3];
    static GLfloat   gltb_transform[4][4];
 
    static GLuint    gltb_width;
    static GLuint    gltb_height;
 
    static GLint     gltb_button = -1;
    static GLboolean gltb_tracking = GL_FALSE;
    static GLboolean gltb_animate = GL_TRUE;
 
 
    static void
    _gltbPointToVector(int x, int y, int width, int height, float v[3])
    {
      float d, a;
 
      /* project x, y onto a hemi-sphere centered within width, height. */
      v[0] = (2.0 * x - width) / width;
      v[1] = (height - 2.0 * y) / height;
      d = sqrt(v[0] * v[0] + v[1] * v[1]);
      v[2] = cos((3.14159265 / 2.0) * ((d < 1.0) ? d : 1.0));
      a = 1.0 / sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
      v[0] *= a;
      v[1] *= a;
      v[2] *= a;
    }
 
    static void
    _gltbAnimate(void)
    {
      glutPostRedisplay();
    }
 
    void
    _gltbStartMotion(int x, int y, int button, int time)
    {
      assert(gltb_button != -1);
 
      gltb_tracking = GL_TRUE;
      gltb_lasttime = time;
      _gltbPointToVector(x, y, gltb_width, gltb_height, gltb_lastposition);
    }
 
    void
    _gltbStopMotion(int button, unsigned time)
    {
      assert(gltb_button != -1);
 
      gltb_tracking = GL_FALSE;
 
      if (time - gltb_lasttime < GLTB_TIME_EPSILON && gltb_animate) {
          glutIdleFunc(_gltbAnimate);
      } else {
        gltb_angle = 0;
        if (gltb_animate)
          glutIdleFunc(0);
      }
    }
 
    void
    gltbAnimate(GLboolean animate)
    {
      gltb_animate = animate;
    }
 
    void
    gltbInit(GLuint button)
    {
      gltb_button = button;
      gltb_angle = 0.0;
 
      /* put the identity in the trackball transform */
      glPushMatrix();
      glLoadIdentity();
      glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)gltb_transform);
      glPopMatrix();
    }
 
    void
    gltbMatrix(void)
    {
      assert(gltb_button != -1);
 
      glPushMatrix();
      glLoadIdentity();
      glRotatef(gltb_angle, gltb_axis[0], gltb_axis[1], gltb_axis[2]);
      glMultMatrixf((GLfloat*)gltb_transform);
      glGetFloatv(GL_MODELVIEW_MATRIX, (GLfloat*)gltb_transform);
      glPopMatrix();
 
      glMultMatrixf((GLfloat*)gltb_transform);
    }
 
    void
    gltbReshape(int width, int height)
    {
      assert(gltb_button != -1);
 
      gltb_width  = width;
      gltb_height = height;
    }
 
    void
    gltbMouse(int button, int state, int x, int y)
    {
      assert(gltb_button != -1);
 
      if (state == GLUT_DOWN && button == gltb_button)
        _gltbStartMotion(x, y, button, glutGet(GLUT_ELAPSED_TIME));
      else if (state == GLUT_UP && button == gltb_button)
        _gltbStopMotion(button, glutGet(GLUT_ELAPSED_TIME));
    }
 
    void
    gltbMotion(int x, int y)
    {
      GLfloat current_position[3], dx, dy, dz;
 
      assert(gltb_button != -1);
 
      if (gltb_tracking == GL_FALSE)
        return;
 
      _gltbPointToVector(x, y, gltb_width, gltb_height, current_position);
 
      /* calculate the angle to rotate by (directly proportional to the
         length of the mouse movement) */
      dx = current_position[0] - gltb_lastposition[0];
      dy = current_position[1] - gltb_lastposition[1];
      dz = current_position[2] - gltb_lastposition[2];
      gltb_angle = 90.0 * sqrt(dx * dx + dy * dy + dz * dz);
 
      /* calculate the axis of rotation (cross product) */
      gltb_axis[0] = gltb_lastposition[1] * current_position[2] - 
                   gltb_lastposition[2] * current_position[1];
      gltb_axis[1] = gltb_lastposition[2] * current_position[0] - 
                   gltb_lastposition[0] * current_position[2];
      gltb_axis[2] = gltb_lastposition[0] * current_position[1] - 
                   gltb_lastposition[1] * current_position[0];
 
      /* XXX - constrain motion */
      gltb_axis[2] = 0;
 
      /* reset for next time */
      gltb_lasttime = glutGet(GLUT_ELAPSED_TIME);
      gltb_lastposition[0] = current_position[0];
      gltb_lastposition[1] = current_position[1];
      gltb_lastposition[2] = current_position[2];
 
      /* remember to draw new position */
      glutPostRedisplay();
    }

If I cast a ray (CTRL + Left_click) at the beginning It works with correct 3D far_point and near_point

Image:

If I rotate the object (use Trackball), the ray doesn't use real eye/far points and I can't figure out why.

Image:

I've tried with GluLookAt instead of glRotatef, but I can't figure out how to move camera with mouse like this trackball does.

[Dark Photon]

So what is your question? Your ray picking isn't working? Or you trackball doesn't work? I'd fix these separately.

First, I don't see a VIEWING transform here. It looks like what you have for a viewing transform is a -3 translate along Z, which isn't that useful, particularly if you want to virtual trackball around an object. Would use gluLookAt as it's intuitive (where are you, where are you looking, what direction is up).

As to the ray picking question, it doesn't look like you're being very deliberate what MODELVIEW transform is active when you do your back transform. What space do you want the backprojected points to be in?

[Gorghino]

My teacher gave me the obj viewer and I need to develop a mesh rayPicking.
Trackball works very well until I cast my ray:
If I load the scene (the object) the ray picks correct farPlane point (in fact I don't see the ray..It's in front of me); If I flip the object with trackball and I retry to cast, the ray is wrong (image 2).

I think that gltbMatrix()'s transformations are the problem but I'm an openGL beginner and I don't know how to fix that. gltbMatrix() should change the View using gltbMotion() data.

About second question, I don't understand you. Are you talking about convert2dto3d()?
I'm following this tutorial (h**p://antongerdelan.net/opengl/raycasting.html) and some other like that to develop my rayPicking.

Here (h**ps://www.dropbox.com/s/7y2f1ainvntfttn/gc_smooth_Aguiari.tar.gz) you can try a demo.
Ctrl+left_click to cast the ray, just the left_click to interact with object.

Sorry about my english too Tnks for the help

[Dark Photon]

About second question, I don't understand you. Are you talking about convert2dto3d()?

I'm talking about when you do "glGetDoublev(GL_MODELVIEW_MATRIX, ... )" followed by "gluUnProject( ... )". What MODELVIEW is active here? Is it the one you want to land your backprojected position in the right space?

[Gorghino]

I'm talking about when you do "glGetDoublev(GL_MODELVIEW_MATRIX, ... )" followed by "gluUnProject( ... )". What MODELVIEW is active here? Is it the one you want to land your backprojected position in the right space?

I don't think so because gltbMatrix() changes that.
The problem is that I can't figure out when/how cast my ray with updated PoV after a rotation with gltb trackball

[Gorghino]

Solved

The problem was the pushMatrix()/popMatrix() at beginning/end in display().