Rendering to FBO, but alpha channel is saturating.

OpenGL OpenGL: Advanced Coding
1

Hi,

I’ve been trying to make simple “Impostors” for the trees in my game. My approach is to simply render three views of the tree into a texture ( using an FBO ) and to render the impostor tree as three intersecting planes with the appropriate texture/coords/etc. For reference, right now I’m only rendering one view of the tree. No point going any further until I’ve got the problems below worked out…

The trouble is, this: My FBO has an alpha channel, and before I render I’m clearing it to black, with alpha of zero ( glClear( 0,0,0,0 )). When I render the foliage for my tree – which is basically a bunch of quads with leafy texturing – the fringe areas of those fragments, where alpha < 1 but > 0, are as far as I can tell saturating the alpha channel of the texture to 1.

The result is that the foliage has a hideous black halo!

Here’s a screenshot of the tree – not an impostor, this is the actual tree.

Here’s a screenshot of the impostor quad.

And, to test my guess that fragments with alpha < 1 still saturate the alpha channel of the target FBO, I disabled the leaf texture when rendering the leaves, and set the alpha of the now-untextured leaf quads to 0.25. You can see that the leaf quads are being blended against the black clear color of the context, but are fully opaque.

And now, some of my code. First, Tree::generateImpostor

void Tree::generateImpostor( void )
{
	_renderingImpostor = true;
	
	_foliage->setImpostorRendering( true );
	
	/*
		For the purposes of development, at first, we'll just set
		up an ortho projection and an FBO to render into, looking
		down the +Y axis. The ortho projection will correspond 
		to the dimension of the tree's AABB and the viewport will
		correspond to the texture size.
	*/
	

	
	int size = 512;
	FrameBufferObject renderTexture( size, size, GL_TEXTURE_2D, true,
									 FrameBufferObject::DepthBuffer | FrameBufferObject::AlphaChannel );

	
	glError();
	
	glMatrixMode( GL_PROJECTION );
	glLoadIdentity();
	glViewport( 0, 0, size, size );
	
	
	AABB bb = aabb() + _foliage->aabb();
	
	/*
		Setup ortho projection to represent tree's cubic (aabb) region
	*/
	glOrtho( bb.minx, bb.maxx, bb.miny, bb.maxy, bb.minz, bb.maxz );
					
	/*
		Create a modelview to look at center of tree
	*/
	
	vec3 eye( bb.center() );
	eye.y = bb.miny;
	
	vec3 at( bb.center() );
			
	mat4 modelview;
	modelview.look_at( eye, at, vec3( 0,0,1 ));
	glMatrixMode( GL_MODELVIEW );
	

	glPushMatrix();
	{
		glLoadMatrixf( modelview.mat );
		
		/*
			Bind render texture
		*/
		glClearColor( 0,0,0,0 );
		renderTexture.begin();
	
		glEnable( GL_DEPTH_TEST );
		
		Light *light = world()->primaryLight();

		
		// render self
		setupState( Normal, SinglePass_Solid, light );
		glEnable( GL_DEPTH_TEST );
		glDisable( GL_FOG );
		display( SinglePass_Solid, light );
		teardownState( Normal, SinglePass_Solid );
		
		// render foliage
		_foliage->setupState( Normal, SinglePass_Solid, light );
		_foliage->display( SinglePass_Solid, light );
		_foliage->teardownState( Normal, SinglePass_Solid );
			
		/*
			End, then bind/unbind to force mipmap generation
		*/
		renderTexture.end();
		
		renderTexture.bind();
		renderTexture.unbind();		
	}
	glPopMatrix();
	
	/*
		Now, release texture from FBO ( since on destruction FBO would 
		otherwise free it ) and bind it to _impostorTexture
	*/
	

	renderTexture.releaseTextureOwnership();
	TextureManager *tm = Application::instance()->textureManager();

	delete _impostorTexture;
	_impostorTexture = tm->take( name() + ":Impostor", renderTexture.textureID(), 
								 renderTexture.width(), renderTexture.height(),
								 renderTexture.components(), renderTexture.format() );

	glError();
	

	_foliage->setImpostorRendering( false );
	_renderingImpostor = false;
}

And here’s my FBO code ( which may very well be the source of the problem )

FrameBufferObject::FrameBufferObject( int width, int height, GLuint target, 
									  bool mipmaps, int flags ):
	_width( width ), _height( height ), _mipmapMaxLevel(0),
	_textureID(0), _depthTextureID(0), _stencilTextureID(0),
	_frameBufferID(0), _depthBufferID(0), _stencilBufferID(0), _target(target),
	_needsClearing( true ), _mipmaps( mipmaps ), 
	_mipmapsDirty( false ), _alphaChannel( flags & AlphaChannel )
{	
	switch( _target )
	{
		case GL_TEXTURE_2D:
		{
			if ( !IsPow2( _width ) &#0124;&#0124; !IsPow2( _height ) )
			{
				Logger::log( LogEntry::Critical, "FrameBufferObject::FrameBufferObject",
							 "GL_TEXTURE_2D requires POT textures ( width: %d height: %d )",
							 _width, _height );
			}

			break;
		}

		case GL_TEXTURE_RECTANGLE_EXT:
		{
			_mipmaps = false;
			break;
		}

		case GL_TEXTURE_CUBE_MAP:
		{
			if ( _width != _height &#0124;&#0124; !IsPow2( _width ) &#0124;&#0124; !IsPow2( _height ) )
			{
				Logger::log( LogEntry::Critical, "FrameBufferObject::FrameBufferObject",
							 "GL_TEXTURE_CUBE_MAP requires POT textures ( width: %d height: %d )",
							 _width, _height );
			}		
			break;
		}	
		
		default:
		{
			Logger::log( LogEntry::Critical, "FrameBufferObject::FrameBufferObject",
						 "FrameBufferObject supports only one of GL_TEXTURE_2D"
						 ", GL_TEXTURE_RECTANGLE_EXT, and GL_TEXTURE_CUBE_MAP. (%d) is not among them.",
						 _target );
						 					 
			break;
		}
	}
	
	GLint format = _alphaChannel ? GL_RGBA : GL_RGB;
		
	switch( _target )
	{
		case GL_TEXTURE_2D:
		{
			if ( _mipmaps )
			{
				_mipmapMaxLevel = lrintf( log2( std::max( _width, _height ))) - 2;		

				/*
					generate the texture; notice we're creating empty textures
					for each mipmap level
				*/

				glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 ); 
				glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );

				glGenTextures( 1, &_textureID );
				glBindTexture( _target, _textureID);
				
				for ( int level = 0; level < _mipmapMaxLevel; level++ )
				{
					glTexImage2D( _target, level, format, _width, _height, 0, format, GL_UNSIGNED_BYTE, NULL );
				}

				glTexParameteri( _target, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR );
				glTexParameteri( _target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
			}
			else
			{
				_mipmapMaxLevel = 0;

				/*
					generate the texture; notice we're creating empty textures
					for each mipmap level
				*/

				glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 ); 
				glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );

				glGenTextures( 1, &_textureID );
				glBindTexture( _target, _textureID);
				
				glTexImage2D( _target, 0, format, _width, _height, 0, format, GL_UNSIGNED_BYTE, NULL );

				glTexParameteri( _target, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
				glTexParameteri( _target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
			}
							
			break;
		}
		
		case GL_TEXTURE_RECTANGLE_EXT:
		{
			_mipmapMaxLevel = 0;
			
			/*
				generate the texture; no mipmaps for RECTANGLE
			*/

			glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 ); 
			glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );

			glGenTextures( 1, &_textureID );
			glBindTexture( _target, _textureID);
			glTexImage2D( _target, 0, format, _width, _height, 0, format, GL_UNSIGNED_BYTE, NULL );

			glTexEnvi( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE );
			glTexParameteri( _target, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
			glTexParameteri( _target, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
			
			break;
		}
		
		case GL_TEXTURE_CUBE_MAP:
		{
			if ( _mipmaps )
			{
				int size = _width;		
				_mipmapMaxLevel = lrintf( log2( size )) - 2;		

				glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 ); 
				glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );

				glGenTextures(1, &_textureID);
				glBindTexture(GL_TEXTURE_CUBE_MAP, _textureID);
				
				glError();

				for ( int level = 0; level < _mipmapMaxLevel; level++ )
				{
					glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_X, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
					glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_X, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
					glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_Y, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
					glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_Y, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
					glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_Z, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
					glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_Z, level, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				}
				
				glError();

				glTexEnvi( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE );
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);
				
				glError();
			}
			else
			{
				int size = _width;		
				_mipmapMaxLevel = 0;

				glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 ); 
				glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );

				glGenTextures(1, &_textureID);
				glBindTexture(GL_TEXTURE_CUBE_MAP, _textureID);
				
				glError();

				glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_X, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_X, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_Y, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_Y, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_Z, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				glTexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_Z, 0, format, size, size, 0, format, GL_UNSIGNED_BYTE, NULL );
				
				glError();

				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
				glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);
				
				glError();
			}

			break;
		}
	}

	/*
		generate the frame buffer
	*/
	glGenFramebuffersEXT( 1, &_frameBufferID );
	glBindFramebufferEXT( GL_FRAMEBUFFER_EXT, _frameBufferID );

	glFramebufferTexture2DEXT( GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT, 
							   _target == GL_TEXTURE_CUBE_MAP ? GL_TEXTURE_CUBE_MAP_POSITIVE_X : _target, 
							   _textureID, 0 );

	CheckFramebufferStatus();

	/*
		Generate attachments
	*/

	if( flags & DepthBuffer )
	{
		glError();
		
		glGenRenderbuffersEXT( 1, &_depthBufferID );
		glBindRenderbufferEXT( GL_RENDERBUFFER_EXT, _depthBufferID );
		glRenderbufferStorageEXT( GL_RENDERBUFFER_EXT, GL_DEPTH_COMPONENT32, _width, _height );
		glError();

		glFramebufferRenderbufferEXT( GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER_EXT, _depthBufferID );

		glError();
		
		CheckFramebufferStatus();
	}

	if( flags & StencilBuffer )
	{
		using namespace PANSICore;
		Logger::log( LogEntry::Normal, "FrameBufferObject",
					 "Stencil buffer not supported as of OS X 10.4.6" );

		/*
		glGenRenderbuffersEXT( 1, &_stencilBufferID );
		glBindRenderbufferEXT( GL_RENDERBUFFER_EXT, _stencilBufferID );
		glRenderbufferStorageEXT( GL_RENDERBUFFER_EXT, GL_STENCIL_INDEX8_EXT, _width, _height );
		glFramebufferRenderbufferEXT( GL_FRAMEBUFFER_EXT, GL_STENCIL_ATTACHMENT_EXT, GL_RENDERBUFFER_EXT, _stencilBufferID );
		CheckFramebufferStatus();
		*/
	}
	
	glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0 );	
}

void FrameBufferObject::begin( void )
{
	if( !_frameBufferID ) 
	{
		return;
	}
	
	glError();
		
	glBindFramebufferEXT( GL_FRAMEBUFFER_EXT, _frameBufferID );

	glError();
		
	if ( _needsClearing )
	{
		/*
			Clear each face of the cubemap
		*/
		if ( _target == GL_TEXTURE_CUBE_MAP )
		{
			for ( int i = 0; i < 6; i++ )
			{
				glFramebufferTexture2DEXT( GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
										   GL_TEXTURE_CUBE_MAP_POSITIVE_X + i, _textureID, 0 );

				glError();
				glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT );
			}
		}
		else
		{
			glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT );
		}

		_needsClearing = false;
	}
}

void FrameBufferObject::end( void )
{
	glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0);
	glFlush();
	
	/*
		Mark mipmaps dirty, provided we're generating mipmaps.
	*/
	_mipmapsDirty = _mipmaps;
}
2

Hi,

sorry if I´m totally wrong here, but in your renderToFBO - code I don´t see GL_BLEND enabled - could this be the reason?

If this is somehow stupid - sorry, I´m not an OpenGL wizard, as I don´t use OpenGL directly.

Regards,

Andreas

3

Oh, yeah, I should have mentioned. The calls setupState and teardownState, which bracket the rendering into the FBO of the Tree object ( solid geometry ) and the foliage take care of enabling and binding and configuring materials, blending, etc etc etc.

4

Try changing the internal format parameter for glTexImage2D from GL_RBGA to GL_RGBA8, it might be substituting a lower precision format with only one bit of alpha.

5

Well, I’m almost too embarrassed to bring this back up, but, well, it turned out that a subtle bug in my scenegraph wasn’t setting up the drawing state correctly for the drawing of the impostor texture itself.

That said, I’ve got interesting new problems. It seems, actually, that the opposite of what I thought was happening is happening.

Here’s a screenshot:

What’s happening, as far as I can tell, is this:
I render the tree trunk will full opacity, the alpha channel of the FBO’s target texture is set to 1 wherever the trunk fragments go. This is correct behavior.

Then, I render the leaves ( a bunch of textured quads ). The leaves are rendered fully opaque, but the texture has an alpha channel. Wherever the alpha < 1, the alpha channel of the FBO seems to be multiplied by the textured fragment’s alpha. E.g., if the fragment’s alpha is 0.5, and the destination value is 1, I’m getting 0.5 as a result. If another, at 0.5 is rendered on top, the alpha goes to 0.25, and so on.

What you can see is that where leaves overlap, my alpha is dropping to zero.

This is what the leaf rendering code looks like. You’ll notice I’m using glBlendFuncSeparate because I thought it might help to use additive alpha coverage here. No dice.

void TreeFoliage::LeafRenderer::display( RenderPass pass )
{
    if ( impostorRendering )
    {
        glEnable( GL_DEPTH_TEST );
        glDepthMask( GL_FALSE );
        glDisable( GL_FOG );
        glDisable( GL_LIGHTING );       
        
        glDisable( GL_ALPHA_TEST );

        glEnable( GL_BLEND );
        glBlendFuncSeparate( GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, 
                             GL_SRC_ALPHA, GL_ONE );
                                                                                                                                                                                                                                                                                                                                                
        glBegin( GL_QUADS );
        
        vec3 ambient( WORLD()->ambientLight() );

        TreeLeafVec::iterator it( leaves.begin() ), end( leaves.end() );
        for ( ; it != end; ++it )
        {
            vec4 color( it->color );
            color.x *= ambient.x;
            color.y *= ambient.y;
            color.z *= ambient.z;
            
            glColor4fv( color.v );
            it->display();
        }
        
        glEnd();

        glBlendFunc( GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA );
        glDepthMask( GL_TRUE );
        glEnable( GL_ALPHA_TEST );
        glEnable( GL_FOG );
        glEnable( GL_LIGHTING );
    }
    else
    {
        //normal rendering...
    }
}