c++ picking function is not working

i have this picking function but it doesn't seems to work. the function should return true if it collide with an object but i开发者_JS百科t return 0; and it never change.

here is my picking function

BOOL D3dDevice::Picking(HWND hWnd, LPDIRECT3DDEVICE9 d3ddev, CXFileEntity *entity)
{
 D3DXVECTOR3 v;
 D3DXMATRIX matProj;
 POINT pt;
 D3DVIEWPORT9 vp;
 GetCursorPos(&pt);
 ScreenToClient(hWnd, &pt);
 d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
 d3ddev->GetViewport(&vp);
 v.x =  ( ( ( 2.0f * pt.x ) / vp.Height  ) - 1 ) / matProj._11;
 v.y = -( ( ( 2.0f * pt.x ) / vp.Width ) - 1 ) / matProj._22;
 v.z =  1.0f;

 D3DXMATRIX m;
 D3DXVECTOR3 rayOrigin,rayDir;
 D3DXMATRIX matView;
 d3ddev->GetTransform(D3DTS_VIEW, &matView);

 D3DXMatrixInverse( &m, NULL, &matView );

 // Transform the screen space pick ray into 3D space
 rayDir.x  = v.x*m._11 + v.y*m._21 + v.z*m._31;
 rayDir.y  = v.x*m._12 + v.y*m._22 + v.z*m._32;
 rayDir.z  = v.x*m._13 + v.y*m._23 + v.z*m._33;
 rayOrigin.x = m._41;
 rayOrigin.y = m._42;
 rayOrigin.z = m._43;

 // Use inverse of matrix
 D3DXMATRIX matInverse, matWorld;
 d3ddev->GetTransform(D3DTS_WORLD, &matWorld);

 // Use inverse of matrix
 D3DXMatrixInverse(&matInverse,NULL,&matWorld);

 // Transform ray origin and direction by inv matrix
 D3DXVECTOR3 rayObjOrigin,rayObjDirection, rayDirection;

 D3DXVec3TransformCoord(&rayObjOrigin,&rayOrigin,&matInverse);
 D3DXVec3TransformNormal(&rayObjDirection,&rayDirection,&matInverse);
 D3DXVec3Normalize(&rayObjDirection,&rayObjDirection);

 BOOL hasHit;
 float distanceToCollision;

 D3DXIntersect(entity->pDrawMesh, &rayObjOrigin, &rayObjDirection, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL);

 return hasHit;
}

note: my pDrawMesh is mutable LPD3DXMESH not LPD3DXBASEMESH would that make a different? UPDATE

BOOL D3dDevice::Picking(HWND hWnd, LPDIRECT3DDEVICE9 d3ddev, CXFileEntity *entity)
{
    D3DXVECTOR3 v;
    D3DXMATRIX matProj;
    POINT pt;
    D3DVIEWPORT9 vp;
    D3DXMATRIX matInverse, matWorld;
    D3DXMATRIX m;
    D3DXVECTOR3 rayOrigin,rayDir;
    D3DXMATRIX matView;
    D3DXVECTOR3 rayObjSpace;
    D3DXVECTOR3 rayObjOrigin,rayObjDirection, rayDirection;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);

    d3ddev->GetTransform(D3DTS_VIEW, &matView);

    // Use inverse of matrix
    d3ddev->GetTransform(D3DTS_WORLD, &matWorld);
    D3DXVECTOR3 vec3( pt.x, pt.y, 1.0f );
    D3DXVec3Unproject( &rayObjSpace, &vec3, &vp, &matProj, &matView, &matWorld );
    // Transform ray origin and direction by inv matrix

    D3DXMATRIX invWorld;
    D3DXMatrixInverse( &invWorld, NULL, &matWorld );

    D3DXVECTOR3 camObjSpace;
    D3DXVECTOR3 camPos(0.0, 0.0, -14.0f);
    D3DXVec3TransformCoord( &camObjSpace, &camPos, &invWorld ); 

    rayDir = rayObjSpace - camObjSpace;

    BOOL hasHit;
    float distanceToCollision;

    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayObjSpace, &rayDir, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    if(hasHit==1)
    {
        PostQuitMessage(0);
    }

    return hasHit;
}

UPDATE 2: Now it doesn't intersect ;/. BOOL D3dDevice::Picking(HWND hWnd, LPDIRECT3DDEVICE9 d3ddev, CXFileEntity *entity, int z) { D3DXVECTOR3 v; POINT pt; D3DVIEWPORT9 vp; D3DXMATRIX matInverse, matWorld, m, matView, matProj;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);
    d3ddev->GetTransform(D3DTS_WORLD, &matWorld);
    d3ddev->GetTransform(D3DTS_VIEW, &matView);

    // Use inverse of matrix
    D3DXVECTOR3 rayPos(pt.x, pt.y,0); // near-plane position
    D3DXVECTOR3 rayDir(pt.x, pt.x,1); // far-plane position
    D3DXVec3Unproject(&rayPos,&rayPos,&vp,&matProj,&matView,&matWorld);
    D3DXVec3Unproject(&rayDir,&rayDir,&vp,&matProj,&matView,&matWorld);
    rayDir -= rayPos; // make a direction from the 2 positions
    D3DXVec3Normalize(&rayDir,&rayDir); // don't know if this is necessary.
    // Transform ray origin and direction by inv matrix

    BOOL hasHit;
    float distanceToCollision;

    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayPos, &rayDir, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    if(hasHit!=0)
        PostQuitMessage(0);

    return hasHit;
}

UPDATE 3: Ok now it always intersect ;/ after changing values in this function

D3DXMatrixPerspectiveFovLH(&matProjection,
                               D3DXToRadian(45),    // the horizontal field of view
                               (FLOAT)Width / (FLOAT)Height, // aspect ratio
                               0.0f,   // the near view-plane
                               1.0f);    // the far view-plane

Ray picking is a fun one. You basically need to start off by converting a screen coordinate into projection space. As DirectX projection space (for x,y) ranges from -1 to 1 in x and y then you need to do the following.

float px = (((float)mousex / SCREEN_WIDTH) * 2.0f) - 1.0f;
float py = -(((float)mousey / SCREEN_HEIGHT) * 2.0f) - 1.0f;

Not py has the minus in front because screen coord go from top to bottom where projection space is the opposite.

You now have your position in projection space. So you need to create a vector along that point. Thankfully thats nice and easy too.

float pz = 1.0f;

Now we can unproject this back into object space:

D3DXVECTOR3 rayObjSpace;
D3DXVec3Unproject( &rayObjSpace, &rayOut, &viewport, &projectionMatrix, &viewMatrix, &worldMatrix );

Finally plug that into D3DXIntersect and you are ready to go :)

Edit: I do realise there is a bug in my explanation above. In fact with D3DXVec3Unproject you can pass in the actual screen coordinate x,y with a z of 1.

So:

D3DXVECTOR3 vec3( mousex, mousey, 1.0f );
D3DXVECTOR3 rayObjSpace;
D3DXVec3Unproject( &rayObjSpace, &vec3, &viewport, &projectionMatrix, &viewMatrix, &worldMatrix );

Much simpler. As it is the view port transform is applied twice.

That said have you actually looked at the ray vectors you have been getting returned? Do they even point towards the object?

Edit 2: Alright I'll be more specific. Now you have the world position of the point you clicked on you can calculate the ray direction by doing the following.

 D3DXMATRIX invWorld;
 D3DXMatrixInverse( &invWorld, NULL, &world );

 D3DXVECTOR3 camObjSpace;
 D3DXVec3TransformCoord( &camObjSpace, &camPos, &invWorld ); 

 D3XVECTOR3 rayDir = rayObjSpace - camObjSpace;

 BOOL bHit = FALSE;
 float distToHit = true;
 HRESULT hr = D3DXIntersect( pMesh, &rayObjSpace, &rayDir, &bHit, NULL, NULL, NULL, &distToHit, NULL, NULL );

The logic that sets hasHit is in D3DXIntersect which we can't see. There are a load of other functions (which look like C functions) that we can't see either.