From TDN

from: http://www.garagegames.com/index.php?sec=mg&mod=resource&page=view&qid=10899

courtessy of Dave Young

While you won't *necessarily* be needing the exact point-on-a-texture support that the above refferenced rescource entails, the underlaying TriRayInfo et al functionality is important to keep in mind, since you _definately_ don't want to be using the beefier functions with every raycast. Besides, if youre going to be dealing with hyper in-dpeth shader tricks, that one adds quite afew more tools to you're arsenal (video on a texture comes to mind, for one).

So. The relevant functionality portions to take away from that particular rescource are going to be:

In engine\math\mathUtils.cpp, after the end of the function getVectorFromAngles, add the following functions. There should still be a final } in the file after these functions and before the end of the file:


F32 getArea( const Point3F* verts, S32 count )
{
	// Vector whose length will be area^2.
	Point3F vec( 0, 0, 0 );
	for ( S32 i = 0; i < count; i++ ) {

      S32 j = i ? i - 1 : count - 1;

      // Add cross_product(pnts[j],pnts[i]).
		vec.x += verts[j].y * verts[i].z - verts[j].z * verts[i].y;
		vec.y += verts[j].z * verts[i].x - verts[j].x * verts[i].z;
		vec.z += verts[j].x * verts[i].y - verts[j].y * verts[i].x;
	}
   
	// Find length of vector, return it.
	return mSqrt( vec.x * vec.x + vec.y * vec.y + vec.z * vec.z ) / 2.0f;
}

Point3F getBarrycentricCoord( const Point3F& point, const Point3F* tri )
{
	// If the area of the triangle is zero... return 0.
	const F32 a = getArea( tri, 3 );

	if ( a <= 0.0f )
      return Point3F( 0, 0, 0 );

	Point3F b;
	{
      Point3F temp[3] =
      {
         tri[1],
         tri[2],
         point
      };
		b.x = getArea( temp, 3 ) / a;
	}
	{
      Point3F temp[3] =
      {
         tri[2],
         tri[0],
         point
      };
		b.y = getArea( temp, 3 ) / a;
	}
	{
      Point3F temp[3] =
      {
         tri[0],
         tri[1],
         point
      };
		b.z = getArea( temp, 3 ) / a;
	}

   return b;
}

bool rayTriangleIntersect( const Point3F& orig, const Point3F& dir,
   const Point3F& vert0, const Point3F& vert1, const Point3F& vert2,
   F32 *t, F32 *u, F32 *v )
{
   const F32 EPSILON = __EQUAL_CONST_F;

   /* find vectors for two edges sharing vert0 */
   Point3F edge1( vert1 -  vert0 );
   Point3F edge2( vert2 -  vert0 );

   /* begin calculating determinant - also used to calculate U parameter */
   Point3F pvec( mCross( dir, edge2 ) );

   /* if determinant is near zero, ray lies in plane of triangle */
   F32 det = mDot( edge1, pvec );

   if (det < EPSILON)
      return 0;

   /* calculate distance from vert0 to ray origin */
   Point3F tvec( orig - vert0 );

   /* calculate U parameter and test bounds */
   *u = mDot( tvec, pvec );
   if (*u < 0.0 || *u > det)
      return false;

   /* prepare to test V parameter */
   Point3F qvec( mCross( tvec, edge1 ) );

   /* calculate V parameter and test bounds */
   *v = mDot( dir, qvec );
   if (*v < 0.0 || *u + *v > det)
      return false;

   /* calculate t, scale parameters, ray intersects triangle */
   F32 inv_det = 1.0 / det;
   *t = mDot( edge2, qvec ) * inv_det;
   *u *= inv_det;
   *v *= inv_det;

   return true;
}


In engine\math\mathUtils.cpp:
Before line 52 which reads:
inline bool isPow2(const U32 number) { return (number & (number - 1)) == 0; }

Add:

/// Returns the area of a polygon.
F32 getArea( const Point3F* verts, S32 count );

/// Returns the barrycentric coordinate for a point on a triangle.
Point3F getBarrycentricCoord( const Point3F& point, const Point3F* tri );

/// Returns the intersection of a ray and a triangle.
bool rayTriangleIntersect( const Point3F& orig, const Point3F& dir, const Point3F& vert0, const Point3F& vert1, const Point3F& vert2, F32 *t, F32 *u, F32 *v );

In engine\ts\tsMesh.cpp, after:
#include "materials/matInstance.h"
Add:
#include "math/mathUtils.h"

Near line 1987, before function TSMesh::addToHull, add the following function:


bool TSMesh::castRayTri( S32 frame, const Point3F& start, const Point3F& end, TriRayInfo* rayInfo )
{
   // Loop thru the triangles of the first frame.
   const S32 firstVert = vertsPerFrame * frame;

   F32 hit = F32_MAX;

   Point3F dir( end - start );
   dir.normalizeSafe();

   for ( S32 i=0; i < primitives.size(); i++ )
   {
      const TSDrawPrimitive& draw = primitives[i];

      AssertFatal( draw.matIndex & TSDrawPrimitive::Indexed,"TSMesh::castRayTri, got non-indexed primitive!" );

      // gonna depend on what kind of primitive it is...
      if ( ( draw.matIndex & TSDrawPrimitive::TypeMask ) == TSDrawPrimitive::Triangles )
      {
         for ( S32 j=0; j < draw.numElements; j+=3 )
         {
            const U32 idx0 = indices[draw.start+j+0] + firstVert;
            const U32 idx1 = indices[draw.start+j+1] + firstVert;
            const U32 idx2 = indices[draw.start+j+2] + firstVert;

            F32 h, u, v;
            if ( MathUtils::rayTriangleIntersect( start, dir, verts[idx2], verts[idx1], 
               verts[idx0], &h, &v, &u ) && h < hit ) 
            {
               hit = h;

               rayInfo->distance = hit;
               rayInfo->point = start + ( dir * hit );
               rayInfo->material = draw.matIndex & TSDrawPrimitive::MaterialMask;

               Point3F temp[3] =
               {
                  verts[idx0],
                  verts[idx1],
                  verts[idx2]
               };
               Point3F bc = MathUtils::getBarrycentricCoord( rayInfo->point, temp );

               rayInfo->uv = Point2F( 
                  (bc.x * tverts[idx0].x) + (bc.y * tverts[idx1].x) + (bc.z * tverts[idx2].x ),
                  (bc.x * tverts[idx0].y) + (bc.y * tverts[idx1].y) + (bc.z * tverts[idx2].y ) );
            }
         }
      }
      else
      {
         AssertFatal( ( draw.matIndex & TSDrawPrimitive::Strip ) == TSDrawPrimitive::Strip, "TSMesh::castRayTri, unexpected primitive type!" );

         U32 idx0 = indices[draw.start + 0] + firstVert;
         U32 idx1;
         U32 idx2 = indices[draw.start + 1] + firstVert;
         U32* nextIdx = &idx1;
         for ( S32 j = 2; j < draw.numElements; j++ )
         {
            *nextIdx = idx2;
            nextIdx = (U32*) ( (dsize_t)nextIdx ^ (dsize_t)&idx0 ^ (dsize_t)&idx1);
            idx2 = indices[draw.start + j] + firstVert;

            F32 h, u, v;
            if ( MathUtils::rayTriangleIntersect( start, dir, verts[idx2], verts[idx1], 
               verts[idx0], &h, &v, &u ) && h < hit )
            {
               hit = h;

               rayInfo->distance = hit;
               rayInfo->point = start + ( dir * hit );
               rayInfo->material = draw.matIndex & TSDrawPrimitive::MaterialMask;

               Point3F temp[3] =
               {
                  verts[idx0],
                  verts[idx1],
                  verts[idx2]
               };
               Point3F bc = MathUtils::getBarrycentricCoord( rayInfo->point, temp );

               rayInfo->uv = Point2F( 
                  (bc.x * tverts[idx0].x) + (bc.y * tverts[idx1].x) + (bc.z * tverts[idx2].x ),
                  (bc.x * tverts[idx0].y) + (bc.y * tverts[idx1].y) + (bc.z * tverts[idx2].y ) );
            }
         }
      }
   }

   return hit < F32_MAX;
}

In engine\ts\tsMesh.h, after:
#include "sceneGraph/sceneState.h"
Add:
#include "sim/sceneObject.h"


Near line 72, before:

class TSMesh
{

Add:

struct TriRayInfo : public RayInfo
{
Point2F uv;
};

Near line 200, after:
virtual bool castRay(S32 frame, const Point3F & start, const Point3F & end, RayInfo * rayInfo);

Add:
virtual bool castRayTri(S32 frame, const Point3F & start, const Point3F & end, TriRayInfo * rayInfo);

EDIT: To get DTS material detection you need to add the following as well to tsCollision.cpp

After :

1. include "sim/sceneObject.h"

Add:

1. include "materials/materialList.h"
2. include "materials/matInstance.h"

Then in TSShapeInstance::castRay, after: rayInfo->point += a;

Add:

     rayInfo->material = 0;

     // Find the material object id for the texture id
     // that TSMesh::castRay() set for us.
     if ( mMaterialList ) 
     {
        MatInstance* matInst = mMaterialList->getMaterialInst( rayInfo->material );
        if ( matInst && matInst->getMaterial() ) 
           rayInfo->material = matInst->getMaterial()->getId();
     }