utils: pxrad: allow self shadowing with "no shadows" spawnflag in vertexlighting (#258)

Author: ncuxonaT

utils/pxrad/trace.cpp

@@ -648,7 +648,7 @@ static void LinkEdict( entity_t *ent, modtype_t modtype, const char *modname, in
 	}
 
 	// don't link into world if shadow was disabled
-	if(( ent->modtype == mod_studio || ent->modtype == mod_alias ) && !FBitSet( flags, FMESH_CAST_SHADOW ))
+	if(( ent->modtype == mod_studio || ent->modtype == mod_alias ) && !FBitSet( flags, FMESH_CAST_SHADOW ) && !FBitSet( flags, FMESH_SELF_SHADOW ))
 		return;
 
 #ifdef HLRAD_RAYTRACE
@@ -1059,8 +1059,23 @@ static void ClipToLinks( areanode_t *node, moveclip_t *clip, bool stop_on_first_
 
 		touch = ENTITY_FROM_AREA( l );
 
-		if( touch == clip->ignore )
-			continue;
+		//hack for self shadowing without casting shadows
+		if( touch->modtype == mod_studio || touch->modtype == mod_alias )
+		{
+			tmesh_t	*mesh = (tmesh_t *)touch->cache;
+
+			if( touch == clip->ignore )
+			{
+				if( !FBitSet( mesh->flags, FMESH_SELF_SHADOW ) )
+					continue;
+			}
+			else
+				if( !FBitSet( mesh->flags, FMESH_CAST_SHADOW ) )
+					continue;			
+		}
+		else
+			if( touch == clip->ignore )
+				continue;
 
 		if( !BoundsIntersect( clip->boxmins, clip->boxmaxs, touch->absmin, touch->absmax ))
 			continue;

utils/pxrad/vertexlight.cpp

@@ -192,8 +192,7 @@ void BuildVertexLights( int indexnum, int thread = -1 )
 	if( !mesh->verts || mesh->numverts <= 0 )
 		return; 
 
-	if( !FBitSet( mesh->flags, FMESH_SELF_SHADOW ))
-		ignoreent = mapent;
+	ignoreent = mapent;
 
 	GetStylesFromMesh( styles, mesh );
 
@@ -314,8 +313,7 @@ void VertexPatchLights( int indexnum, int threadnum = -1 )
 	if( !mesh->verts || mesh->numverts <= 0 )
 		return; 
 
-	if( !FBitSet( mesh->flags, FMESH_SELF_SHADOW ))
-		ignoreent = mapent;		
+	ignoreent = mapent;		
 
 	GetStylesFromMesh( newstyles, mesh );