From TDN

Contents 1 Engine Code Changes 2 Shapes 3 Terrains 4 Terrain - ToonColors 5 Interiors 6 Preferences

Engine Code Changes

In TGE, there will be no perfect solution for Cel-Shading. True Cel-Shading cannot be achieved without big rendering changes, but the code below is provided to create an outline for the terrain, shapes and interiors. There are some big differences between the DirectX and OpenGL (Using this code). It is recommended that you look into Stencil Shadows, help define this visual style. A word of caution, this code and cel-shading in general is a serious fps hit.

The code presented below is largly from this forum thread:
Topic: cartoon outlining (cartoon rendering part1)

Insert/Replace these lines of code for each appropriate area.

Shapes

in ts/tsMesh.cc
in TSMesh::render after:

S32 drawType = getDrawType(draw.matIndex>>30);
glDrawElements(drawType,draw.numElements,GL_UNSIGNED_SHORT,&indices[draw.start]);
   }

add

   /*cell shading*/
   bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
   
   if (doOutline) {
      bool oldlighting=glIsEnabled(GL_LIGHTING);
      glDisable(GL_LIGHTING);
      F32 oldLineWidth;
      glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
      F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);
      glLineWidth(lineWidth);
      glCullFace(GL_FRONT);
      glPolygonMode (GL_BACK, GL_LINE);
      glDepthFunc(GL_LEQUAL);
      bool old2dtex=glIsEnabled(GL_TEXTURE_2D);
      glDisable(GL_TEXTURE_2D);
      F32 oldcolor[4];
      glGetFloatv(GL_CURRENT_COLOR,oldcolor);
      glColor4f(0.0f,0.0f,0.0f,1.0f); // Outline Color
      for (S32 i=0; i<primitives.size(); i++)
      {
         TSDrawPrimitive & draw = primitives[i];
         AssertFatal(draw.matIndex & TSDrawPrimitive::Indexed,
                  "TSMesh::render: rendering of non-indexed meshes no longer supported");

         S32 drawType = getDrawType(draw.matIndex>>30);
         glDrawElements(drawType,draw.numElements,GL_UNSIGNED_SHORT,&indices[draw.start]);
     }

      glColor4fv(oldcolor);
      if (old2dtex) glEnable(GL_TEXTURE_2D);
      glDepthFunc(GL_LEQUAL);
      glLineWidth(oldLineWidth);
      glColor3f(1.0f,1.0f,1.0f);
      glCullFace(GL_BACK);
      glPolygonMode (GL_BACK, GL_FILL);
      if (oldlighting) glEnable(GL_LIGHTING);
   }

   /* end cell shading*/

Terrains

in terrain/terrRender.cc
in TerrainRender::renderXFCache

void TerrainRender::renderXFCache()
{
U32 count = 0;

bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
F32 oldLineWidth;
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);

while (count < mXFIndexCount)
{
U32 mode = mXFIndexBuffer[count];
U32 vertexCount = mXFIndexBuffer[count + 1];

// Cartoon Outline
if(doOutline)
{
glHint (GL_LINE_SMOOTH_HINT, GL_NICEST); // Use The Good Calculations
glEnable (GL_LINE_SMOOTH);
glPolygonMode (GL_BACK, GL_LINE);
glLineWidth(lineWidth);

glDisable(GL_CULL_FACE);

glColor3f(0.0f,0.0f,0.0f);
glDisable(GL_TEXTURE_2D);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
glEnable(GL_TEXTURE_2D);
glEnable(GL_CULL_FACE);
glLineWidth(oldLineWidth);
glPolygonMode (GL_BACK, GL_FILL);

glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
else
{
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
// End Cartoon Outline

count += vertexCount + 2;
}
}

Terrain - ToonColors

in scenegraph\sceneLighting.cc
void SceneLighting::TerrainProxy::lightVector(LightInfo * light)

Replace this:

Point3F lightDir = light->mDirection;

With This:

Point3F lightDir;//= light->mDirection;

lightDir.x = 0.001f;
lightDir.y = 0.001f;
lightDir.z = -80.0f;

Theres also an if else blocked (search for shadowed? to find it)

It should start like this:

if(height >= intHeight)
{
}
else{
}

Replace with this:

// Cell Shading
bool toonColors = Con::getBoolVariable("$Pref::toonColors",true);
bool shadowCheck = true;

if (!toonColors)
{
shadowCheck = height >= intHeight;
}

if(shadowCheck)//height >= intHeight) // non shadowed
{
U32 idx = (xmask + (ymask << blockShift)) << 2;

Point3F normal = pNormals[bi] * normTable[idx++];
normal += pNormals[binext] * normTable[idx++];
normal += pNextNormals[binext] * normTable[idx++];
normal += pNextNormals[bi] * normTable[idx];
normal.normalize();

nextHeightArray[y] = height;
F32 colorScale = -mDot(normal, lightDir);
if (colorScale < 0.0)
{
colorScale = 0.0;
}

if (toonColors) 
{
col.red = 5.0 * colorScale;
col.green = 5.0 * colorScale;
col.blue = 5.0 * colorScale;

}
else
{
col += ambient + lightColor * colorScale * lightScale[lsi];
}
// End Cell Shading

}
else // Shadowed
{
nextHeightArray[y] = intHeight;
col += ambient;
}

Interiors

in interior/interiorRender.cc
in void Interior::render after:

   if (smRenderMode != 0) {
      PROFILE_START(IRO_DebugRender);
      debugRender(pMaterials, instanceHandle);
      PROFILE_END();
      return;
   }

add

   //Cel-Shading outline

   bool doOutline=Con::getBoolVariable("$Pref::Interior::renderOutline",false);

   if (doOutline) {
      F32 oldLineWidth=1;
      glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
       F32 lineWidth=Con::getFloatVariable("$Pref::Interior::OutlineWidth",2);
       glLineWidth(lineWidth);

      // Base textures
      glBlendFunc(GL_ONE, GL_ZERO);
      glDisable(GL_TEXTURE_2D);
      glColor3f(0, 0, 0);
      for (U32 i = 0; i < sgActivePolyListSize; i++) {

        const Surface& rSurface = mSurfaces[sgActivePolyList[i]];
         glBegin(GL_LINE_LOOP);
         glVertex3fv(mPoints[mWindings[rSurface.windingStart]].point);
         S32 skip = rSurface.windingStart + 1;
         while (skip < (rSurface.windingStart + rSurface.windingCount)) {

           glVertex3fv(mPoints[mWindings[skip]].point);
           skip += 2;
         }

         skip -= 1;
         while (skip > rSurface.windingStart) {
           if (skip < (rSurface.windingStart + rSurface.windingCount))
             glVertex3fv(mPoints[mWindings[skip]].point);
           skip -= 2;
         }
         glEnd();
      }
      glEnable(GL_TEXTURE_2D);
      glLineWidth(oldLineWidth);
     //return;
   }
   //End cartoon outline

Preferences

The following are the required preferences that the code uses.

Cel-Shading: Render: Outline: (true/false)

$Pref::renderOutline = "true";

Cel-Shading: Render: Outline: Size: (0-10)

$Pref::outlineWidth = "3";

Cel-Shading: Render: Outline: Interiors: (true/false)

$Pref::Interior::renderOutline = "true";

Cel-Shading: Render: Outline: Interiors: Size: (0-10)

$Pref::Interior::outlineWidth = "3";

Cel-Shading: Render: ToonColors: (true/false)

$Pref::toonColors = "true";