Want to be taught to mouse pick with raycasting step-by-step (C++, FreeGLUT, OpenGL)

I’ve been going around for a few days trying to find a tutorial that doesn’t absolutely confuse me on how to mouse pick using raycasting but have had no luck.
Most of the time I get confused as to what I’m doing because I don’t get enough information on what it’s currently showing me.

So I’ve come here to ask someone to teach me(In detail if possible so I can understand as much as possible) how to mouse pick using raycasting.

I’m just here to learn so I can get better at my programming because I’m not even close to mastering it like a lot of you answering my questions lately; I don’t have a lot of time during my day to spend on research for things like this so please be patient with me if I don’t know something that you do.

If it helps anything I’m learning OpenGL from here: Lazy Foo' Productions - OpenGL Tutorials

these tutorials are relatively old, i would recommend to get after this one:

ok, it isnt raycasting, but it’s in my opinion the best & easiest way to pick objects

there are 2 other tutorials for picking, but apparently they use a physics library for that:

if your graphics card supports openGL 4.5, here is the simplest example (using MRT):
main.h


#pragma once

#include <GL/glew.h>
#include <GLFW/glfw3.h>


#include <glm/glm.hpp>
#include <glm/gtx/transform.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/quaternion.hpp>
#include <glm/gtx/quaternion.hpp>
#include <glm/gtx/rotate_vector.hpp>
#include <glm/gtc/type_ptr.hpp>


class Main
{
public:

	static Main& Instance();
	virtual ~Main();

	int run();
	void render();

	unsigned int TrackedID() const;

protected:

	Main();

	GLFWwindow* m_window;
	friend void cursor_position_callback(GLFWwindow* window, double xpos, double ypos);
	friend void window_size_callback(GLFWwindow* window, int width, int height);
	friend void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods);
	friend void scroll_callback(GLFWwindow* window, double xoffset, double yoffset);

	glm::ivec2 m_displaysize;
	glm::ivec2 m_cursor;

	glm::vec3 m_camera{ 0, 1, 5 };

	// framebuffer
	struct
	{
		glm::ivec2 size;
		unsigned int ID;
		unsigned int color0, color1, depth;
	} m_framebuffer;
	void SetupFramebuffer();

	unsigned int m_program;
	void SetupShader();

	unsigned int m_vao, m_vbo;
	void SetupVertexArray();

	// picking
	unsigned int m_trackedID;

};


main.cpp


#include "Main.h"
#include <iostream>


int main(void)
{
	return Main::Instance().run();
}



void window_size_callback(GLFWwindow* window, int width, int height)
{
	Main::Instance().m_displaysize = glm::ivec2(width, height);
	Main::Instance().SetupFramebuffer();
}


void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods)
{
	if (action == GLFW_PRESS || action == GLFW_REPEAT)
	{
		if (key == GLFW_KEY_W)
			Main::Instance().m_camera.z -= 0.2f;
		if (key == GLFW_KEY_A)
			Main::Instance().m_camera.x -= 0.2f;
		if (key == GLFW_KEY_S)
			Main::Instance().m_camera.z += 0.2f;
		if (key == GLFW_KEY_D)
			Main::Instance().m_camera.x += 0.2f;
	}
}


void cursor_position_callback(GLFWwindow* window, double xpos, double ypos)
{
	// invert y-coordinate
	Main::Instance().m_cursor = glm::ivec2(xpos, Main::Instance().m_displaysize.y - ypos);
}


void scroll_callback(GLFWwindow* window, double xoffset, double yoffset)
{
	//if (yoffset > 0)
	//	std::cout << "scrolled up" << std::endl;
	//else
	//	std::cout << "scrolled down" << std::endl;
}


void mouse_button_callback(GLFWwindow* window, int button, int action, int mods)
{
	if (action == GLFW_PRESS)
	{
		if (button == GLFW_MOUSE_BUTTON_LEFT)
			std::cout << "clicked on " << Main::Instance().TrackedID() << std::endl;
	}
}


glm::vec4 IntegerToColor(int i)
{
	int r = (i & 0x000000FF) >> 0;
	int g = (i & 0x0000FF00) >> 8;
	int b = (i & 0x00FF0000) >> 16;
	int a = (i & 0xFF000000) >> 24;
	return glm::vec4(r / 255.0, g / 255.0, b / 255.0, a / 255.0);
}



int Main::run()
{
	/* Loop until the user closes the window */
	while (!glfwWindowShouldClose(m_window))
	{
		/* Render here */
		render();

		/* Swap front and back buffers */
		glfwSwapBuffers(m_window);

		/* Poll for and process events */
		glfwPollEvents();
	}

	return 0;
}


void Main::render()
{
	// prepare framebuffer
	// ---------------------------------------------------------------------------------------------
	glBindFramebuffer(GL_FRAMEBUFFER, m_framebuffer.ID);

	// clear scene buffer
	glClearColor(0.3f, 0.3f, 0.3f, 0);
	glDrawBuffer(GL_COLOR_ATTACHMENT0);
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

	// clear picking buffer (should be totally black)
	glClearColor(0, 0, 0, 0);
	glDrawBuffer(GL_COLOR_ATTACHMENT1);
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

	// MRT: render into both layers
	unsigned int drawbuffers[] = { GL_COLOR_ATTACHMENT0 , GL_COLOR_ATTACHMENT1 };
	glDrawBuffers(2, drawbuffers);
	// ---------------------------------------------------------------------------------------------

	

	// render scene
	// ---------------------------------------------------------------------------------------------
	static float angle(0);
	angle += 0.01f;
	
	glEnable(GL_DEPTH_TEST);

	glUseProgram(m_program);

	glUniformMatrix4fv(glGetUniformLocation(m_program, "View"), 1, false, glm::value_ptr(glm::lookAt(m_camera, glm::vec3(0, 0, 0), glm::vec3(0, 1, 0))));
	glUniformMatrix4fv(glGetUniformLocation(m_program, "Projection"), 1, false, glm::value_ptr(glm::perspective(3.14f / 4, (float)m_displaysize.x / m_displaysize.y, 0.1f, 100.0f)));

	glBindVertexArray(m_vao);
	glEnableVertexAttribArray(0);
	glEnableVertexAttribArray(1);

	// first triangle
	glUniformMatrix4fv(glGetUniformLocation(m_program, "Model"), 1, false, glm::value_ptr(glm::translate(glm::vec3(-1, 0, 0)) * glm::rotate(angle, glm::vec3(0, 1, 0))));
	glUniform4fv(glGetUniformLocation(m_program, "IDcolor"), 1, &IntegerToColor(123)[0]);

	glDrawArrays(GL_TRIANGLES, 0, 3);

	// second triangle
	glUniformMatrix4fv(glGetUniformLocation(m_program, "Model"), 1, false, glm::value_ptr(glm::translate(glm::vec3(0, 0, 0)) * glm::rotate(angle, glm::vec3(0, 1, 0))));
	glUniform4fv(glGetUniformLocation(m_program, "IDcolor"), 1, &IntegerToColor(246)[0]);

	glDrawArrays(GL_TRIANGLES, 0, 3);

	// third triangle
	glUniformMatrix4fv(glGetUniformLocation(m_program, "Model"), 1, false, glm::value_ptr(glm::translate(glm::vec3(1, 0, 0)) * glm::rotate(angle, glm::vec3(0, 1, 0))));
	glUniform4fv(glGetUniformLocation(m_program, "IDcolor"), 1, &IntegerToColor(777)[0]);

	glDrawArrays(GL_TRIANGLES, 0, 3);

	glDisableVertexAttribArray(0);
	glDisableVertexAttribArray(1);

	glUseProgram(0);
	// ---------------------------------------------------------------------------------------------


	// picking
	// ---------------------------------------------------------------------------------------------
	unsigned char pixeldata[4];

	// reading pixel data at current cursor position ...
	glReadBuffer(GL_COLOR_ATTACHMENT1);	// read from second framebuffer layer
	glReadPixels(m_cursor.x, m_cursor.y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixeldata);

	// convert pixel color back to (int)ID ...
	m_trackedID = (pixeldata[0] << 0) | (pixeldata[1] << 8) | (pixeldata[2] << 16) | (pixeldata[3] << 24);
	// ---------------------------------------------------------------------------------------------



	// copy to system-framebuffer
	// ---------------------------------------------------------------------------------------------
	glBindFramebuffer(GL_READ_FRAMEBUFFER, m_framebuffer.ID);
	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);

	glReadBuffer(GL_COLOR_ATTACHMENT0);	// default scene
	//glReadBuffer(GL_COLOR_ATTACHMENT1);	// that would show you the IDcolors

	glClearColor(1, 1, 1, 1);		// everything that is white is out of our framebuffers area
	glClear(GL_COLOR_BUFFER_BIT);

	glBlitFramebuffer(
		0, 0, m_framebuffer.size.x, m_framebuffer.size.y,
		0, 0, m_framebuffer.size.x, m_framebuffer.size.y,
		GL_COLOR_BUFFER_BIT,
		GL_NEAREST);
	// ---------------------------------------------------------------------------------------------
}


unsigned int Main::TrackedID() const
{
	// ID at cursor position
	return m_trackedID;
}


Main::Main()
{
	/* Initialize the library */
	if (!glfwInit())
		exit(-1);

	/* Create a windowed mode window and its OpenGL context */
	m_displaysize = glm::ivec2(640, 480);
	m_window = glfwCreateWindow(m_displaysize.x, m_displaysize.y, "Hello World", NULL, NULL);
	if (!m_window)
	{
		glfwTerminate();
		exit(-1);
	}

	/* Make the window's context current */
	glfwMakeContextCurrent(m_window);

	// set callback functions
	glfwSetWindowSizeCallback(m_window, window_size_callback);
	glfwSetKeyCallback(m_window, key_callback);
	glfwSetCursorPosCallback(m_window, cursor_position_callback);
	glfwSetScrollCallback(m_window, scroll_callback);
	glfwSetMouseButtonCallback(m_window, mouse_button_callback);

	/* Init GLEW* */
	if (glewInit() != GLEW_OK)
	{
		glfwTerminate();
		exit(-2);
	}

	// create framebuffer
	glGenRenderbuffers(1, &m_framebuffer.color0);	// default, for scene
	glGenRenderbuffers(1, &m_framebuffer.color1);	// for picking
	glGenRenderbuffers(1, &m_framebuffer.depth);		// depth buffer
	glGenFramebuffers(1, &m_framebuffer.ID);

	SetupFramebuffer();
	SetupShader();
	SetupVertexArray();
}


Main & Main::Instance()
{
	static Main instance;
	return instance;
}


Main::~Main()
{
	glfwTerminate();
}


void Main::SetupFramebuffer()
{
	m_framebuffer.size = m_displaysize;

	// resize viewport
	glViewport(0, 0, m_framebuffer.size.x, m_framebuffer.size.y);

	// allocate memory
	glBindRenderbuffer(GL_RENDERBUFFER, m_framebuffer.color0);
	glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA, m_framebuffer.size.x, m_framebuffer.size.y);
	glBindRenderbuffer(GL_RENDERBUFFER, m_framebuffer.color1);
	glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA, m_framebuffer.size.x, m_framebuffer.size.y);
	glBindRenderbuffer(GL_RENDERBUFFER, m_framebuffer.depth);
	glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT32, m_framebuffer.size.x, m_framebuffer.size.y);
	glBindRenderbuffer(GL_RENDERBUFFER, 0);

	// attach to framebuffer
	glBindFramebuffer(GL_FRAMEBUFFER, m_framebuffer.ID);
	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, m_framebuffer.color0);
	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT1, GL_RENDERBUFFER, m_framebuffer.color1);
	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, m_framebuffer.depth);

	// show errors
	GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
	if (status != GL_FRAMEBUFFER_COMPLETE)
	{
		std::cout << "ERROR: Framebuffer incomplete: ";
		if (status == GL_FRAMEBUFFER_UNDEFINED)
			std::cout << "undefined framebuffer";
		if (status == GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT)
			std::cout << "a necessary attachment is uninitialized";
		if (status == GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT)
			std::cout << "no attachments";
		if (status == GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER)
			std::cout << "incomplete draw buffer";
		if (status == GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER)
			std::cout << "incomplete read buffer";
		if (status == GL_FRAMEBUFFER_UNSUPPORTED)
			std::cout << "combination of attachments is not supported";
		if (status == GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE)
			std::cout << "number if samples for all attachments does not match";
		std::cout << std::endl;
	}

	glBindFramebuffer(GL_FRAMEBUFFER, 0);
}


void Main::SetupShader()
{
	// shaders
	std::string vertexshader =
		"#version 400
"\
		"layout(location = 0) in vec3 VertexPosition;
"\
		"layout(location = 1) in vec4 VertexColor;
"\
		"out vec4 color;
"\
		"uniform mat4 Model, View, Projection;
"\
		"void main()
"\
		"{
"\
		"	mat4 MVP = Projection * View * Model;
"\
		"	gl_Position = MVP * vec4(VertexPosition, 1);
"\
		"	color = VertexColor;
"\
		"}
"\
		;


	std::string fragmentshader =
		"#version 400
"\
		"in vec2 TextureCoords;
"\
		"in vec4 color;
"\
		"layout (location = 0) out vec4 FragmentColor0;"\
		"layout(location = 1) out vec4 FragmentColor1;"\
		"uniform vec4 IDcolor;"\
		"void main()
"\
		"{
"\
		"	FragmentColor0 = color;
"\
		"	FragmentColor1 = IDcolor;
"\
		"}
"\
		;


	// reset error log
	GLenum ErrorCheckValue = glGetError();

	// create shaders
	GLuint shadervertexID = glCreateShader(GL_VERTEX_SHADER);
	GLuint shaderfragmentID = glCreateShader(GL_FRAGMENT_SHADER);

	// openGL wants for every shader an array of c-strings as source code
	const char* sourcecodearrayVS[] = { vertexshader.c_str() };
	const char* sourcecodearrayFS[] = { fragmentshader.c_str() };

	// set source code for shaders
	glShaderSource(shadervertexID, 1, sourcecodearrayVS, nullptr);
	glShaderSource(shaderfragmentID, 1, sourcecodearrayFS, nullptr);

	// compile shaders
	glCompileShader(shadervertexID);
	glCompileShader(shaderfragmentID);

	// create program
	m_program = glCreateProgram();

	// attach shaders to program
	glAttachShader(m_program, shadervertexID);
	glAttachShader(m_program, shaderfragmentID);

	// link program
	glLinkProgram(m_program);

	// tag for deletion (takes effect when no shader program uses them anymore)
	glDeleteShader(shadervertexID);
	glDeleteShader(shaderfragmentID);

	// check for errors ...
	ErrorCheckValue = glGetError();
	if (ErrorCheckValue != GL_NO_ERROR)
	{
		std::cout << "ERROR: cannot load shaders:   errorcode = " << ErrorCheckValue << std::endl;
		system("PAUSE");
		exit(-1);
	}
}


void Main::SetupVertexArray()
{
	float vertices[] = {
	//   xyz                        rgba
		0.0f, 0.0f, 0.0f,		 1.0f, 0.0f, 0.0f, 1.0f,
		1.0f, 0.0f, 0.0f,		 0.0f, 1.0f, 0.0f, 1.0f,
		0.0f, 1.0f, 0.0f,		 0.0f, 0.0f, 1.0f, 1.0f,
	};
	
	glGenVertexArrays(1, &m_vao);
	glBindVertexArray(m_vao);

	glGenBuffers(1, &m_vbo);
	glBindBuffer(GL_ARRAY_BUFFER, m_vbo);
	glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);

	glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, sizeof(float) * 7, (void*)(sizeof(float) * 0));
	glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, sizeof(float) * 7, (void*)(sizeof(float) * 3));

	glBindBuffer(GL_ARRAY_BUFFER, 0);
	glBindVertexArray(0);
}


description:
Main class is a singelton class, that means there will only be 1 object of that class, and you cann access that instance from everywhere (without passing it by reference as an argument)

int main()
call that instance, and calls the “run” method on it (which is just the main loop)

window_size_callback()
if you resize the window, this function will be called automatically (by your operating system)
here it updates a potected member “m_displaysize” and rebuilds our custom framebuffer
IMPORTANT: that function it declared as “friend” in Main class, that’s why it can access protected members like displaysize

cursor_position_callback
it just updates the cursor position member of the Main class
NOTE: the y-coordinate is inverted because we want the lower left corner to be the origin (to consistent with openGL)

scroll_callback()
empty

mouse_button_callback()
it just shows on what ID you have clicked
the ID is a member of Main class, it is calculated from the color rendered into the second renderbuffer (GL_COLOR_ATTACHMENT1) / or “second layer” every frame (AFTER you’ve finished rendering your scene)

glm::vec4 IntegerToColor(int i)
converts an integer to a renderable color

int Main::run()
main loop

void Main::render()
the rendering process, first we bind our custom framebuffer and clear it, then we tell opengl to render into 2 specific layers:


// MRT: render into both layers
unsigned int drawbuffers[] = { GL_COLOR_ATTACHMENT0 , GL_COLOR_ATTACHMENT1 };
glDrawBuffers(2, drawbuffers);

then rendering the scene with a special fragment shader, that renders into 2 layers


#version 400
in vec2 TextureCoords;
in vec4 color;

layout (location = 0) out vec4 FragmentColor0;
layout(location = 1) out vec4 FragmentColor1;

uniform vec4 IDcolor;   // this color will be rendered into the second layer

void main()
{
	FragmentColor0 = color;
	FragmentColor1 = IDcolor;
};

by uploading the “uniform vec4 IDcolor” we tell opengl what color (or better: what integer ID) should be used for the current object that is being rendered
here we are just rendering 3 rotating triangles …

after we’re done with the scene, the picking process takes part:
by callling “glReadBuffer(GL_COLOR_ATTACHMENT1);” we tell opengl to read from our second layer (in which we rendered the ID (transformed to a color))
then we rebuild the ID (saved in m_trackedID) by bit-shifting the rgba-components of the ID color

finally, we have to copy the scene we rendered into OUR framebuffer (which cant be displayed on the window) to the system-framebuffer (which is shown on the window every frame)
glBindFramebuffer(GL_READ_FRAMEBUFFER, m_framebuffer.ID); tells opengl to read from our framebuffer
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); tells opengl to draw into the system-famebuffer (0 = system-famebuffer)
glBlitFramebuffer() does the actual copy

done !!!

constructor Main::Main() declared protected
means that there cant be any instance created from outside that class, that guarantees that we will only have 1 instance of that class in the whole application
the constructor just sets up the window, our double-layered framebuffer, our special shader for that framebuffer, and the vertexarray (triangle model)


if your graphics card doesnt support framebuffers or 4.0 shaders, you have to build 2 “normal” shaders, 1 for the scene rendering and 1 for the ID rendering
whe you begin the render process, you have to render first the whole scene with the ID shader, then do the picking, then clear & render the scene again with the normal color shader


later if you have many objects to render in your scene, it is better to remove the uniform vec4 IDcolor from the fragment shader and implement it as a “instanced” vertex attribute (as well as the model matrix)

1 Like

Thank you so much john_connor you’ve been very helpful when I’ve asked questions here!!!