Toon shader with Texture. Can this be optimized?

Hi all,

I am quite new to OpenGL, I have managed after long trial and error to integrate Nehe’s Cel-Shading rendering with my Model loaders, and have them drawn using the Toon shade and outline AND their original texture at the same time. The result is actually a very nice Cel Shading effect of the model texture, but it is havling the speed of the program, it’s quite very slow even with just 3 models on screen…

Since the result was kind of hacked together, I am thinking that maybe I am performing some extra steps or extra rendering tasks that maybe are not needed, and are slowing down the game?
Something unnecessary that maybe you guys could spot?

Both MD2 and 3DS loader have an InitToon() function called upon creation to load the shader

initToon(){

	int i;														// Looping Variable ( NEW )
	char Line[255];												// Storage For 255 Characters ( NEW )
	float shaderData[32][3];									// Storate For The 96 Shader Values ( NEW )
	FILE *In = fopen ("Shader.txt", "r");						// Open The Shader File ( NEW )

	if (In)														// Check To See If The File Opened ( NEW )
	{
		for (i = 0; i < 32; i++)								// Loop Though The 32 Greyscale Values ( NEW )
		{
			if (feof (In))										// Check For The End Of The File ( NEW )
				break;

			fgets (Line, 255, In);								// Get The Current Line ( NEW )

			shaderData[i][0] = shaderData[i][1] = shaderData[i][2] = float(atof (Line)); // Copy Over The Value ( NEW )
		}

		fclose (In);											// Close The File ( NEW )
	}

	else
		return false;											// It Went Horribly Horribly Wrong ( NEW )

	glGenTextures (1, &shaderTexture[0]);						// Get A Free Texture ID ( NEW )

	glBindTexture (GL_TEXTURE_1D, shaderTexture[0]);			// Bind This Texture. From Now On It Will Be 1D ( NEW )

	// For Crying Out Loud Don't Let OpenGL Use Bi/Trilinear Filtering! ( NEW )
	glTexParameteri (GL_TEXTURE_1D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);	
	glTexParameteri (GL_TEXTURE_1D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

	glTexImage1D (GL_TEXTURE_1D, 0, GL_RGB, 32, 0, GL_RGB , GL_FLOAT, shaderData);	// Upload ( NEW )


}

This is the drawing for the animated MD2 model:

MD2Model::drawToon() {
	float		outlineWidth	= 3.0f;								// Width Of The Lines ( NEW )
	float		outlineColor[3]	= { 0.0f, 0.0f, 0.0f };				// Color Of The Lines ( NEW )
	
// ORIGINAL PART OF THE FUNCTION	

	//Figure out the two frames between which we are interpolating
	int frameIndex1 = (int)(time * (endFrame - startFrame + 1)) + startFrame;
	if (frameIndex1 > endFrame) {
		frameIndex1 = startFrame;
	}
	
	int frameIndex2;
	if (frameIndex1 < endFrame) {
		frameIndex2 = frameIndex1 + 1;
	}
	else {
		frameIndex2 = startFrame;
	}
	
	MD2Frame* frame1 = frames + frameIndex1;
	MD2Frame* frame2 = frames + frameIndex2;
	
	//Figure out the fraction that we are between the two frames
	float frac =
		(time - (float)(frameIndex1 - startFrame) /
		 (float)(endFrame - startFrame + 1)) * (endFrame - startFrame + 1);
	

// I ADDED THESE FROM NEHE'S TUTORIAL FOR FIRST PASS (TOON SHADE)

	glHint (GL_LINE_SMOOTH_HINT, GL_NICEST);				// Use The Good Calculations ( NEW )
	glEnable (GL_LINE_SMOOTH);
	// Cel-Shading Code //
	glEnable (GL_TEXTURE_1D);									// Enable 1D Texturing ( NEW )
	glBindTexture (GL_TEXTURE_1D, shaderTexture[0]);			// Bind Our Texture ( NEW )

	glColor3f (1.0f, 1.0f, 1.0f);								// Set The Color Of The Model ( NEW )

// ORIGINAL DRAWING CODE

	//Draw the model as an interpolation between the two frames
	glBegin(GL_TRIANGLES);
	for(int i = 0; i < numTriangles; i++) {
		MD2Triangle* triangle = triangles + i;
		for(int j = 0; j < 3; j++) {
			MD2Vertex* v1 = frame1->vertices + triangle->vertices[j];
			MD2Vertex* v2 = frame2->vertices + triangle->vertices[j];
			Vec3f pos = v1->pos * (1 - frac) + v2->pos * frac;
			Vec3f normal = v1->normal * (1 - frac) + v2->normal * frac;
			if (normal[0] == 0 && normal[1] == 0 && normal[2] == 0) {
				normal = Vec3f(0, 0, 1);
			}
			glNormal3f(normal[0], normal[1], normal[2]);
			
			MD2TexCoord* texCoord = texCoords + triangle->texCoords[j];
			glTexCoord2f(texCoord->texCoordX, texCoord->texCoordY);
			glVertex3f(pos[0], pos[1], pos[2]);
		}
	}
	glEnd();

// ADDED THESE FROM NEHE'S FOR SECOND PASS (OUTLINE)

	glDisable (GL_TEXTURE_1D);									// Disable 1D Textures ( NEW )


	glEnable (GL_BLEND);									// Enable Blending ( NEW )
		glBlendFunc(GL_SRC_ALPHA,GL_ONE_MINUS_SRC_ALPHA);		// Set The Blend Mode ( NEW )

		glPolygonMode (GL_BACK, GL_LINE);						// Draw Backfacing Polygons As Wireframes ( NEW )
		glLineWidth (outlineWidth);								// Set The Line Width ( NEW )

		glCullFace (GL_FRONT);									// Don't Draw Any Front-Facing Polygons ( NEW )

		glDepthFunc (GL_LEQUAL);								// Change The Depth Mode ( NEW )

		glColor3fv (&outlineColor[0]);							// Set The Outline Color ( NEW )


// HERE I AM PARSING THE VERTICES AGAIN (NOT IN THE ORIGINAL FUNCTION) FOR THE OUTLINE AS PER NEHE'S TUT

		glBegin (GL_TRIANGLES);									// Tell OpenGL What We Want To Draw
		for(int i = 0; i < numTriangles; i++) {
		MD2Triangle* triangle = triangles + i;
		for(int j = 0; j < 3; j++) {
			MD2Vertex* v1 = frame1->vertices + triangle->vertices[j];
			MD2Vertex* v2 = frame2->vertices + triangle->vertices[j];
			Vec3f pos = v1->pos * (1 - frac) + v2->pos * frac;
			Vec3f normal = v1->normal * (1 - frac) + v2->normal * frac;
			if (normal[0] == 0 && normal[1] == 0 && normal[2] == 0) {
				normal = Vec3f(0, 0, 1);
			}
			glNormal3f(normal[0], normal[1], normal[2]);
			
			MD2TexCoord* texCoord = texCoords + triangle->texCoords[j];
			glTexCoord2f(texCoord->texCoordX, texCoord->texCoordY);
			glVertex3f(pos[0], pos[1], pos[2]);
		}
	}
		glEnd ();												// Tell OpenGL We've Finished

		glDepthFunc (GL_LESS);									// Reset The Depth-Testing Mode ( NEW )

		glCullFace (GL_BACK);									// Reset The Face To Be Culled ( NEW )

		glPolygonMode (GL_BACK, GL_FILL);						// Reset Back-Facing Polygon Drawing Mode ( NEW )

		glDisable (GL_BLEND);	

Whereas this is the drawToon function in the 3DS loader

void Model_3DS::drawToon()
{

	float		outlineWidth	= 3.0f;								// Width Of The Lines ( NEW )
	float		outlineColor[3]	= { 0.0f, 0.0f, 0.0f };				// Color Of The Lines ( NEW )

//ORIGINAL CODE

	if (visible)
	{
	glPushMatrix();

		// Move the model
		glTranslatef(pos.x, pos.y, pos.z);

		// Rotate the model
		glRotatef(rot.x, 1.0f, 0.0f, 0.0f);
		glRotatef(rot.y, 0.0f, 1.0f, 0.0f);
		glRotatef(rot.z, 0.0f, 0.0f, 1.0f);

		glScalef(scale, scale, scale);

		// Loop through the objects
		for (int i = 0; i < numObjects; i++)
		{
			// Enable texture coordiantes, normals, and vertices arrays
			if (Objects[i].textured)
				glEnableClientState(GL_TEXTURE_COORD_ARRAY);
			if (lit)
				glEnableClientState(GL_NORMAL_ARRAY);
			glEnableClientState(GL_VERTEX_ARRAY);

			// Point them to the objects arrays
			if (Objects[i].textured)
				glTexCoordPointer(2, GL_FLOAT, 0, Objects[i].TexCoords);
			if (lit)
				glNormalPointer(GL_FLOAT, 0, Objects[i].Normals);
			glVertexPointer(3, GL_FLOAT, 0, Objects[i].Vertexes);

			// Loop through the faces as sorted by material and draw them
			for (int j = 0; j < Objects[i].numMatFaces; j ++)
			{
				// Use the material's texture
				Materials[Objects[i].MatFaces[j].MatIndex].tex.Use();


// AFTER THE TEXTURE IS APPLIED I INSERT THE TOON FUNCTIONS HERE (FIRST PASS)



					glHint (GL_LINE_SMOOTH_HINT, GL_NICEST);				// Use The Good Calculations ( NEW )
					glEnable (GL_LINE_SMOOTH);
					// Cel-Shading Code //
					glEnable (GL_TEXTURE_1D);									// Enable 1D Texturing ( NEW )
					glBindTexture (GL_TEXTURE_1D, shaderTexture[0]);			// Bind Our Texture ( NEW )

						glColor3f (1.0f, 1.0f, 1.0f);								// Set The Color Of The Model ( NEW )



				glPushMatrix();

					// Move the model
					glTranslatef(Objects[i].pos.x, Objects[i].pos.y, Objects[i].pos.z);

					// Rotate the model
								
					glRotatef(Objects[i].rot.z, 0.0f, 0.0f, 1.0f);
					glRotatef(Objects[i].rot.y, 0.0f, 1.0f, 0.0f);
					glRotatef(Objects[i].rot.x, 1.0f, 0.0f, 0.0f);

					// Draw the faces using an index to the vertex array
					glDrawElements(GL_TRIANGLES, Objects[i].MatFaces[j].numSubFaces, GL_UNSIGNED_SHORT, Objects[i].MatFaces[j].subFaces);

				glPopMatrix();
			}



				glDisable (GL_TEXTURE_1D);									// Disable 1D Textures ( NEW )


// THIS IS AN ADDED SECOND PASS A THE VERTICES FOR THE OUTLINE


	glEnable (GL_BLEND);									// Enable Blending ( NEW )
		glBlendFunc(GL_SRC_ALPHA,GL_ONE_MINUS_SRC_ALPHA);		// Set The Blend Mode ( NEW )

		glPolygonMode (GL_BACK, GL_LINE);						// Draw Backfacing Polygons As Wireframes ( NEW )
		glLineWidth (outlineWidth);								// Set The Line Width ( NEW )

		glCullFace (GL_FRONT);									// Don't Draw Any Front-Facing Polygons ( NEW )

		glDepthFunc (GL_LEQUAL);								// Change The Depth Mode ( NEW )

		glColor3fv (&outlineColor[0]);							// Set The Outline Color ( NEW )

		for (int j = 0; j < Objects[i].numMatFaces; j ++)
			{
		glPushMatrix();

					// Move the model
					glTranslatef(Objects[i].pos.x, Objects[i].pos.y, Objects[i].pos.z);

					// Rotate the model
									glRotatef(Objects[i].rot.z, 0.0f, 0.0f, 1.0f);
					glRotatef(Objects[i].rot.y, 0.0f, 1.0f, 0.0f);
					glRotatef(Objects[i].rot.x, 1.0f, 0.0f, 0.0f);

					// Draw the faces using an index to the vertex array
					glDrawElements(GL_TRIANGLES, Objects[i].MatFaces[j].numSubFaces, GL_UNSIGNED_SHORT, Objects[i].MatFaces[j].subFaces);

				glPopMatrix();


		}

				glDepthFunc (GL_LESS);									// Reset The Depth-Testing Mode ( NEW )

		glCullFace (GL_BACK);									// Reset The Face To Be Culled ( NEW )

		glPolygonMode (GL_BACK, GL_FILL);						// Reset Back-Facing Polygon Drawing Mode ( NEW )

		glDisable (GL_BLEND);

glPopMatrix();
}

Finally this is the tex.Use() function that loads a BMP texture and somehow gets blended perfectly with the Toon shading

void GLTexture::Use()
{
	glEnable(GL_TEXTURE_2D);								// Enable texture mapping
	glBindTexture(GL_TEXTURE_2D, texture[0]);				// Bind the texture as the current one

}

Didn’t you post this already?

well I at that point I was asking advice on how to achieve this, now I wanted to share this experiment where I have achieved the effect though the performance could de with an increase… I was pointed in the direction of using Vertex Buffer Objects as apparently I am using immediate rendering which is bad and deprecated, I guess that will be my next step…

I have had a shot at replacing the immediate drawing with glDrawArrays, I do this by storing all the vertices in a std::vector and then read them with glDrawArrays…which is supposed to be faster, but I am getting an even worse performance… Does this function look right/efficient?

    for(int i = 0; i < numTriangles; i++) {
        MD2Triangle* triangle = triangles + i;
        for(int j = 0; j < 3; j++) {
            MD2Vertex* v1 = frame1->vertices + triangle->vertices[j];
            MD2Vertex* v2 = frame2->vertices + triangle->vertices[j];
            Vec3f pos = v1->pos * (1 - frac) + v2->pos * frac;
            Vec3f normal = v1->normal * (1 - frac) + v2->normal * frac;
            if (normal[0] == 0 && normal[1] == 0 && normal[2] == 0) {
                normal = Vec3f(0, 0, 1);
            }


            normals.push_back(normal[0]);
            normals.push_back(normal[1]);
            normals.push_back(normal[2]);

            MD2TexCoord* texCoord = texCoords + triangle->texCoords[j];
            textCoords.push_back(texCoord->texCoordX);
            textCoords.push_back(texCoord->texCoordY);

            vertices.push_back(pos[0]);
            vertices.push_back(pos[1]);
            vertices.push_back(pos[2]);
        }

    }


    glEnableClientState(GL_NORMAL_ARRAY);
    glEnableClientState(GL_TEXTURE_COORD_ARRAY);
    glEnableClientState(GL_VERTEX_ARRAY);

    glNormalPointer(GL_FLOAT, 0, &normals[0]);
    glTexCoordPointer(2, GL_FLOAT, 0, &textCoords[0]); 
    glVertexPointer(3, GL_FLOAT, 0, &vertices[0]);



    glDrawArrays(GL_TRIANGLES, 0, vertices.size()/3);


    glDisableClientState(GL_VERTEX_ARRAY);  // disable vertex arrays
    glDisableClientState(GL_TEXTURE_COORD_ARRAY);
    glDisableClientState(GL_NORMAL_ARRAY);

    vertices.clear();
    textCoords.clear();
    normals.clear();

Keeping in mind that vertices positions need to be recalculated at each frame since they are animated models…

Thanks for any help that you may be able to provide

Keeping in mind that vertices positions need to be recalculated at each frame since they are animated models…

Then maybe you need to find a better way to animate models. You’re doing vertex animation. There’s no reason you couldn’t just send both position arrays to the vertex shader and do the blending there. Just pass it the frac value, and it can do all this stuff for you.

Also, you never said how you were testing the performance of any of this.

Thanks for your reply. I am just checking the frame rate really, I get a very good one without the toon effect, but with it (since I am doing the interpolating loop twice, it goes down to roughly 25… With glDrawArrays it’s 10-15… And these are all very low-poly models, less than 2000 vertices, and there would be only 3 or 4 maximum in one scene, that’s why I originally settled for the simple MD2 format…

How would I go about sending those vertices to the vertex shader like you said?

Here’s another try, this one uses one interleaved dynamic array instead of three std::vectors… Better performance, from 15FPS to 25FPS now with a few models on screen… THis is the new one, including the Cel-shading code…is this as optimized as I can get without using vertex shaders?

// ... calculate frame position....


int vCount = 0;

    for(int i = 0; i < numTriangles; i++) {

        //Calculate vertices interpolation
        MD2Triangle* triangle = triangles + i;
        for(int j = 0; j < 3; j++) {
            MD2Vertex* v1 = frame1->vertices + triangle->vertices[j];
            MD2Vertex* v2 = frame2->vertices + triangle->vertices[j];
            Vec3f pos = v1->pos * (1 - frac) + v2->pos * frac;
            Vec3f normal = v1->normal * (1 - frac) + v2->normal * frac;
            if (normal[0] == 0 && normal[1] == 0 && normal[2] == 0) {
                normal = Vec3f(0, 0, 1);
            }
            
            displayList[vCount] = normal[0];
            displayList[vCount+1] = normal[1];
            displayList[vCount+2] = normal[2];
            vCount+=3;

            MD2TexCoord* texCoord = texCoords + triangle->texCoords[j];
            displayList[vCount] = texCoord->texCoordX;
            displayList[vCount+1] = texCoord->texCoordY;
            vCount+=2;

            displayList[vCount] = pos[0];
            displayList[vCount+1] = pos[1];
            displayList[vCount+2] = pos[2];
            vCount +=3;
        }

    }


    glEnableClientState(GL_NORMAL_ARRAY);
    glEnableClientState(GL_TEXTURE_COORD_ARRAY);
        glEnableClientState(GL_VERTEX_ARRAY);

        glNormalPointer(GL_FLOAT, 8*sizeof(float), &displayList[0]);
        glTexCoordPointer(2, GL_FLOAT, 8*sizeof(float), &displayList[3]); 
        glVertexPointer(3, GL_FLOAT, 8*sizeof(float), &displayList[5]);

    // Cel-Shading Code  = Shade
    glHint (GL_LINE_SMOOTH_HINT, GL_FASTEST);                // Use The Good Calculations ( NEW )
    glEnable (GL_LINE_SMOOTH);
    glEnable (GL_TEXTURE_1D);                                    // Enable 1D Texturing ( NEW )
    glBindTexture (GL_TEXTURE_1D, shaderTexture[0]);            // Bind Our Texture ( NEW )
    glColor3f (1.0f, 1.0f, 1.0f);                                // Set The Color Of The Model ( NEW )

    //PASS 1
    glDrawArrays(GL_TRIANGLES, 0, numTriangles * 3);


    glDisable (GL_TEXTURE_1D);                                    // Disable 1D Textures ( NEW )
    

        //Cel-Shading Code = Outline
    glEnable (GL_BLEND);                                    // Enable Blending ( NEW )
    glBlendFunc(GL_SRC_ALPHA,GL_ONE_MINUS_SRC_ALPHA);        // Set The Blend Mode ( NEW )

    glPolygonMode (GL_BACK, GL_LINE);                        // Draw Backfacing Polygons As Wireframes ( NEW )
    glLineWidth (outlineWidth);                                // Set The Line Width ( NEW )
    glCullFace (GL_FRONT);                                    // Don't Draw Any Front-Facing Polygons ( NEW )
    glDepthFunc (GL_LEQUAL);                                // Change The Depth Mode ( NEW )
    glColor3fv (&outlineColor[0]);                            // Set The Outline Color ( NEW )

    //PASS 2
    glDrawArrays(GL_TRIANGLES, 0, numTriangles * 3);


    glDepthFunc (GL_LESS);                                    // Reset The Depth-Testing Mode ( NEW )
    glCullFace (GL_BACK);                                    // Reset The Face To Be Culled ( NEW )
    glPolygonMode (GL_BACK, GL_FILL);                        // Reset Back-Facing Polygon Drawing Mode ( NEW )
    glDisable (GL_BLEND);    

    glDisableClientState(GL_VERTEX_ARRAY);  // disable vertex arrays
    glDisableClientState(GL_TEXTURE_COORD_ARRAY);
    glDisableClientState(GL_NORMAL_ARRAY);

Thanks again for your helpful suggestions

As say Alphonse, it’s certainly better to make the blending computation into the vertex shader.

And if yours models are relatively smalls, I think that to group them into the same vertex array, instead to make a glEnableClientState/gl*Pointer/glDisableClientState cycle for each of them, can have a big impact on performances.

And only make the glEnable/glHint/glBlendFunc/glCullFace/glDepthFunc and others glBindTexture calls at the initialisation, cf. to not make exactely the same thing for each object since they use always the same “constants” states
(I see that you use two passes, so you have to use one different initialisation for each pass, but where each pass display alls models of courses)