The values that get passed into that function will be pixel coordinates then the functions would call gluPickMatrix((GLdouble) x, (GLdouble) (viewport[3] -y), 1.0, 1.0, viewport);
where 1.0 and 1.0 is the size I chose of the selection region.
Not sure if you know anything about selecting objects in OpenGL but I managed to make this from looking around at websites and using the Red and Blue books.
This is the code I have for selection.
//in my windows message area
case WM_LBUTTONDOWN:
MouseFunc(SF_LEFT_BUTTON, SF_DOWN, LOWORD(lParam), HIWORD(lParam));
break;
//functions
void MouseFunc( int button, int state, int x, int y )
{
if( button == SF_LEFT_BUTTON && state == SF_UP )
leftMouseDown = false;
if( button == SF_RIGHT_BUTTON && state == SF_UP )
rightMouseDown = false;
if( button == SF_LEFT_BUTTON && state == SF_DOWN )
leftMouseDown = true;
if( !leftMouseDown || rightMouseDown )
return;
GLuint BUFSIZE = 512;
GLuint selectBuf[BUFSIZE];
GLint hits;
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
glSelectBuffer(BUFSIZE, selectBuf);
glMatrixMode(GL_PROJECTION);
glPushMatrix();
glRenderMode(GL_SELECT);
glInitNames();
glPushName(0);
glLoadIdentity();
gluPickMatrix((GLdouble) x, (GLdouble) (viewport[3] -y), 1.0, 1.0, viewport);
ORTHO_WIDTH = ORTHO_HEIGHT*((GLdouble)WINDOW_WIDTH/(GLdouble)WINDOW_HEIGHT);
glOrtho(0, ORTHO_WIDTH, 0, ORTHO_HEIGHT, -1, 20);
glMatrixMode(GL_MODELVIEW);
glPushMatrix();
//DrawFunc(GL_SELECT);
hits = glRenderMode(GL_RENDER);
glPopMatrix();
glMatrixMode(GL_PROJECTION);
glPopMatrix();
glMatrixMode(GL_MODELVIEW);
glFlush();
objectSelected = ProcessHits(hits, selectBuf);
if( objectSelected >= 0 )
cout << "selected object: " << objectSelected << endl;
}
GLint ProcessHits(GLint hits, GLuint buffer[])
{
if( hits == 0 )
return -1;
GLuint lowestDepth = buffer[1];
GLint selectedObject = buffer[3];
for( sui i = 1; i < hits; i++ )
{
if(buffer[(i*4)+1] < lowestDepth)
{
lowestDepth …