Show stream from webcam and more

Hello,

I am trying to show images from my webcam on background and rotate a teapot on the foreground.
Here is my code:

#include "Displayer.h"
 
 
static const char *glsl_drawtex_vertshader_src =
        "void main(void)
"
        "{
"
        "    gl_Position = gl_Vertex;
"
        "    gl_TexCoord[0].xy = gl_MultiTexCoord0.xy;
"
        "}
";
 
static const char *glsl_drawtex_fragshader_src =
        "#version 130
"
        "uniform usampler2D texImage;
"
        "void main()
"
        "{
"
        "   vec4 c = texture(texImage, gl_TexCoord[0].xy);
"
        "    gl_FragColor = c / 255.0;
"
        "}
";
 
 
 
static const char *glsl_draw_fragshader_src =
        "#version 130
"
        "out uvec4 FragColor;
"
        "void main()
"
        "{"
        "  FragColor = uvec4(gl_Color.xyz * 255.0, 255.0);
"
        "}
";
 
 
static const char *glsl_draw_vert =
        "void main()
"
        "{
"
        "    gl_Position = ftransform();
"
        "}
";
 
 
 
static const char *glsl_draw_frag =
        "void main()
"
        "{
"
        "    gl_FragColor = vec4(0.4,0.4,0.8,1.0);
"
        "}
";
 
 
__global__ void DisplayerKernel(u_int8_t *u8x3_Data, unsigned int *g_odata, int w,int h)
{
 
    int x = blockIdx.x*blockDim.x;
    int y = blockIdx.y*blockDim.y;
    int xglobal = w - x+threadIdx.x;
    int yglobal = h - y+threadIdx.y;
 
 
    if(xglobal>=w || yglobal >=h)
        return;
 
    int iptr        =  xglobal+ yglobal*w;
    int iptr_dst    =  (xglobal-1)+ (h-yglobal-1)*w;
 
    int iptr_rgb    =  xglobal*3+ yglobal*w*3;
    u_int8_t u8_R   = u8x3_Data[iptr_rgb];
    u_int8_t u8_G   = u8x3_Data[iptr_rgb+1];
    u_int8_t u8_B   = u8x3_Data[iptr_rgb+2];
    u_int8_t u8_A   = 0x00;
 
    g_odata[iptr_dst]   = (int(u8_A)<<24) | (int(u8_R)<<16) | (int(u8_G)<<8) | int(u8_B);
 
}
 
 
Displayer::Displayer(int i_windowheight, int i_windowwidth, int i_imageheight, int i_imagewidth):
    // Blending_GPU(iwidth,iheight)
    i_windowwidth_(i_windowwidth),
    i_windowheight_(i_windowheight),
    i_imagewidth_(i_imagewidth),
    i_imageheight_(i_imageheight)
{
    //Init the GL context
    init();
}
 
 
Displayer::~Displayer()
{
    // free device memory
    std::cout << "Destroy Displayer " << std::endl;
}
 
 
 
void Displayer::display(u_int8_t* u8x3_rgbImage_Device)
{
 
 
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
 
    // run the Cuda kernel
    unsigned int *out_data;
    out_data = cuda_dest_resource;
 
    dim3 blocks(ceil((float)i_imagewidth_ / ( BLOCK_SIZE_X)), ceil((float)i_imageheight_ / BLOCK_SIZE_Y));
    dim3 threads(BLOCK_SIZE_X, BLOCK_SIZE_Y);
 
    DisplayerKernel<<<blocks,threads>>>(u8x3_rgbImage_Device,out_data,i_imagewidth_,i_imageheight_);
    checkCudaErrors(cudaGetLastError());
 
    // We want to copy cuda_dest_resource data to the texture
    // map buffer objects to get CUDA device pointers
    cudaArray *texture_ptr;
    checkCudaErrors(cudaGraphicsMapResources(1, &cuda_tex_result_resource, 0));
    checkCudaErrors(cudaGraphicsSubResourceGetMappedArray(&texture_ptr, cuda_tex_result_resource, 0, 0));
 
    int inum_texels = i_imagewidth_ * i_imageheight_;
    int inum_values = inum_texels * 4;
    int isize_tex_data = sizeof(GLubyte) * inum_values;
    checkCudaErrors(cudaMemcpyToArray(texture_ptr, 0, 0, cuda_dest_resource, isize_tex_data, cudaMemcpyDeviceToDevice));
 
    checkCudaErrors(cudaGraphicsUnmapResources(1, &cuda_tex_result_resource, 0));
 
 
    ////////////////////////////////////
    ///////////////////////////////////////////////////
    /// Display texture
    glBindTexture(GL_TEXTURE_2D, tex_cudaResult);
    glEnable(GL_TEXTURE_2D);
//    glDisable(GL_DEPTH_TEST);
    //glDisable(GL_LIGHTING);
//    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);
 
    /*glMatrixMode(GL_PROJECTION);
    glPushMatrix();
    glLoadIdentity();
    //glOrtho(-1.0, 1.0, -1.0, 1.0, -1.0, 10.0);
 
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
 
    glViewport(0, 0, i_windowwidth_, i_windowheight_);
*/
    glUseProgram(shDrawTex);
    GLint id = glGetUniformLocation(shDrawTex, "texImage");
    glUniform1i(id, 0); // texture unit 0 to "texImage"
 
 
    glBegin(GL_QUADS);
    glTexCoord2f(0.0, 0.0);
    glVertex3f(-1.0, -1.0, 0.0);
    glTexCoord2f(1.0, 0.0);
    glVertex3f(1.0, -1.0, 0.0);
    glTexCoord2f(1.0, 1.0);
    glVertex3f(1.0, 1.0, 0.0);
    glTexCoord2f(0.0, 1.0);
    glVertex3f(-1.0, 1.0, 0.0);
    glEnd();
 
    glMatrixMode(GL_PROJECTION);
    glPopMatrix();
 
    glDisable(GL_TEXTURE_2D);
 
    cudaDeviceSynchronize();
 
    // flip backbuffer
    //glutSwapBuffers();
 
 
 
    glUseProgram(shDraw);
//    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
    glLoadIdentity();
    gluLookAt(0.0,0.0,5.0,
              0.0,0.0,-1.0,
              0.0f,1.0f,0.0f);
 
    static float a= 0;
    //glLightfv(GL_LIGHT0, GL_POSITION, lpos);
    glRotatef(a,0,1,1);
    glutSolidTeapot(1);
    a+=0.1;
 
    glutSwapBuffers();
    glUseProgram(0);
 
}
 
void printShaderInfoLog(GLuint obj)
{
    int infologLength = 0;
    int charsWritten  = 0;
    char *infoLog;
 
    glGetShaderiv(obj, GL_INFO_LOG_LENGTH,&infologLength);
 
    if (infologLength > 0)
    {
        infoLog = (char *)malloc(infologLength);
        glGetShaderInfoLog(obj, infologLength, &charsWritten, infoLog);
        printf("%s
",infoLog);
        free(infoLog);
    }
}
 
void Displayer::init()
{
 
 
 
 
 
 
    setenv ("DISPLAY", ":0", 0);
 
    /////////////////////////
    // Create GL context
    char *myargv[1];
    int myargc=1;
    myargv [0]= "BGEDisplayer";
    glutInit(&myargc, myargv);
    glutInitDisplayMode(GLUT_RGBA | GLUT_ALPHA | GLUT_DOUBLE | GLUT_DEPTH);
    glutInitWindowSize(i_windowwidth_, i_windowheight_);
    iGLUTWindowHandle = glutCreateWindow("CUDA OpenGL post-processing");
 
 
        glEnable(GL_DEPTH_TEST);
        glClearColor(1.0,1.0,1.0,1.0);
        glEnable(GL_CULL_FACE);
 
 
    // initialize necessary OpenGL extensions
    glewInit();
 
    /*glClearColorIuiEXT(128,128,128,255);
 
    glDisable(GL_DEPTH_TEST);*/
 
    // viewport
    glViewport(0, 0, i_windowwidth_, i_windowheight_);
 
    // projection
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPerspective(60.0, (GLfloat)i_windowwidth_ / (GLfloat) i_windowheight_, 0.1f, 10.0f);
    glMatrixMode(GL_MODELVIEW);
 
 
 
    /////////////////////////////
    //// createTextureDst
    // create a texture
    glGenTextures(1, &tex_cudaResult);
    glBindTexture(GL_TEXTURE_2D, tex_cudaResult);
 
    // set basic parameters
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8UI_EXT, i_imagewidth_, i_imageheight_, 0, GL_RGBA_INTEGER_EXT, GL_UNSIGNED_BYTE, NULL);
    // register this texture with CUDA
    checkCudaErrors(cudaGraphicsGLRegisterImage(&cuda_tex_result_resource, tex_cudaResult,
                                                GL_TEXTURE_2D, cudaGraphicsMapFlagsWriteDiscard));
 
    // load shader programs
    shDraw      = compileGLSLprogram(glsl_draw_vert, glsl_draw_frag);
    shDrawTex   = compileGLSLprogram(glsl_drawtex_vertshader_src, glsl_drawtex_fragshader_src);
 
 
    //     initCUDABuffers()
    // set up vertex data parameter
    int inum_texels = i_imagewidth_ * i_imageheight_;
    int inum_values = inum_texels * 4;
    int isize_tex_data = sizeof(GLubyte) * inum_values;
    checkCudaErrors(cudaMalloc((void **)&cuda_dest_resource, isize_tex_data));
 
 
 
 
}
 
 
 
GLuint Displayer::compileGLSLprogram(const char *vertex_shader_src, const char *fragment_shader_src)
{
    GLuint v, f, p = 0;
 
    p = glCreateProgram();
 
    if (vertex_shader_src)
    {
        v = glCreateShader(GL_VERTEX_SHADER);
        glShaderSource(v, 1, &vertex_shader_src, NULL);
        glCompileShader(v);
 
        // check if shader compiled
        GLint compiled = 0;
        glGetShaderiv(v, GL_COMPILE_STATUS, &compiled);
 
        if (!compiled)
        {
            //#ifdef NV_REPORT_COMPILE_ERRORS
            char temp[256] = "";
            glGetShaderInfoLog(v, 256, NULL, temp);
            printf("Vtx Compile failed:
%s
", temp);
            //#endif
            glDeleteShader(v);
            return 0;
        }
        else
        {
            glAttachShader(p,v);
        }
    }
 
    if (fragment_shader_src)
    {
        f = glCreateShader(GL_FRAGMENT_SHADER);
        glShaderSource(f, 1, &fragment_shader_src, NULL);
        glCompileShader(f);
 
        // check if shader compiled
        GLint compiled = 0;
        glGetShaderiv(f, GL_COMPILE_STATUS, &compiled);
 
        if (!compiled)
        {
            //#ifdef NV_REPORT_COMPILE_ERRORS
            char temp[256] = "";
            glGetShaderInfoLog(f, 256, NULL, temp);
            printf("frag Compile failed:
%s
", temp);
            //#endif
            glDeleteShader(f);
            return 0;
        }
        else
        {
            glAttachShader(p,f);
        }
    }
 
    glLinkProgram(p);
 
    int infologLength = 0;
    int charsWritten  = 0;
 
    glGetProgramiv(p, GL_INFO_LOG_LENGTH, (GLint *)&infologLength);
 
    if (infologLength > 0)
    {
        char *infoLog = (char *)malloc(infologLength);
        glGetProgramInfoLog(p, infologLength, (GLsizei *)&charsWritten, infoLog);
        printf("Shader compilation error: %s
", infoLog);
        free(infoLog);
    }
 
    return p;
}

I don’t understand, I see the image from my camera but not the teapot.

Can someone give me tips about that?

Best regard.

Check for GL errors. As-is, you’re program is generating them. Find/fix those first.

Comment out your camera image render. Can you see your teapot?

Also, to rule out depth occlusion, disable DEPTH_TEST for your rendering.

Try rendering both the image and teapot with the same MODELVIEW and PROJECTION matrices active.

Thank you for the answer. Now it seems to work, I have the teopot and the background image.
I just remove the line gluLookAt for display both object. But I had to add that line when I just display the teapot. I don’t understand why.


    //////////////////////////////////
    ///////////////////////////////////////////////////
    /// Display texture
    glBindTexture(GL_TEXTURE_2D, tex_cudaResult);
    glEnable(GL_TEXTURE_2D);
    glDisable(GL_DEPTH_TEST);
    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);

    glMatrixMode(GL_PROJECTION);
    glPushMatrix();
    glLoadIdentity();

    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();


    glUseProgram(shDrawTex);
    GLint id = glGetUniformLocation(shDrawTex, "texImage");
    glUniform1i(id, 0); // texture unit 0 to "texImage"


    glBegin(GL_QUADS);
    glTexCoord2f(0.0, 0.0);
    glVertex3f(-1.0, -1.0, 0.0);
    glTexCoord2f(1.0, 0.0);
    glVertex3f(1.0, -1.0, 0.0);
    glTexCoord2f(1.0, 1.0);
    glVertex3f(1.0, 1.0, 0.0);
    glTexCoord2f(0.0, 1.0);
    glVertex3f(-1.0, 1.0, 0.0);
    glEnd();

    glMatrixMode(GL_PROJECTION);
    glPopMatrix();

    glDisable(GL_TEXTURE_2D);

    cudaDeviceSynchronize();

    // flip backbuffer
//    glutSwapBuffers();



    glUseProgram(shDraw);
    glLoadIdentity();
    /*gluLookAt(0.0,0.0,5.0,
              0.0,0.0,-1.0,
              0.0f,1.0f,0.0f);
*/
    static float a= 0;
    //glLightfv(GL_LIGHT0, GL_POSITION, lpos);
    glTranslatef(0,0.5,a);
    glRotatef(a,0,1,1);
    glutSolidTeapot(0.1);
    a+=0.1;

    glutSwapBuffers();
    glUseProgram(0);

Now the problem is I can translate the teapot with the previous code. No translation is saw. At the begining the teapot is display and when the a value is up to a specific value it just vanish.
I would like to be able to translate my teapot, Why can’t there is tese beaviours?