Subtle Checker Boarding Artifact In a custom vertex mesh.

Im getting a very subtle checkerboard artifact along the triangle edges shown by light to normal shading on white.

Im wondering if anyone has seen this before and maybe knows a quick fix?
Im thinking maybe there is some setting im not aware of.
I can’t seem to get rid of it i tried all the sampler states i could think of nothing.
So im thinking its some sort of antialiasing on the edges of the triangles.
Multisample does nothing either.
Nothing in my shader does anything like that, its very simple.

It appears that the triangles edges are drawn either lighter or darker like a sort of soft aliasing.

Hopefully you can see it. Its not visible with textures drawn only when i shade white against the light and surface normals of the mesh, i mean open gl appears to be doing it…

Its not a bag deal with a texture but if that textures really smooth or reflective i can imagine it might become one.

edit basic effect.

Maybe you forgot to normalize your normals in the pixel shader

Humm the normals are all passed in from the vertice data normalize.
By the time it gets to the pixel shader its just interpolated shading colors or white in this instance.
Maybe that’s not a straightforward thing though ?
Humm maybe the normals are messed up somehow but still how would they give off that effect.

//______________________________________________________________
// shader techniques TechniqueB
//______________________________________________________________
VertexShaderOutputB VertexShaderFunctionB(VertexShaderInputB input)
{
    VertexShaderOutputB output;
    output.Position = mul(input.Position, gworldviewprojection);
    output.TexureCoordinateA = input.TexureCoordinateA * input.Color;
    float3 normal = normalize(mul(input.Normal, gworld));
    output.Normal = normal;
    float4 col = input.Color;
    float alpha = col.a;
    float lightIntensity = saturate(dot(normal, lightDir));
    col = ((col *.5 + lightColor *.5) * (lightIntensity * (1 - ambientColor))) + (col * ambientColor);
    col.a = alpha;
    output.Color = col;
    return output;
}

PixelShaderOutputB PixelShaderFunctionB(VertexShaderOutputB input)
{
    PixelShaderOutputB output;
    output.Color = tex2D(TextureSamplerA, input.TexureCoordinateA) * input.Color;
    // just display vertex normal shading
    if (displayVertexNormalShading)
    {
        output.Color = input.Color;
    }
    return output;
}

Heres the full thing if you want to test it. It’s a bit sloppy though
Technique A is only used for the arrows separately here.
Terrain is just the above technique.

//_______________________________________________________________
// RtEffects01.fx
// PNCMT PosNormalColorTexture  test run game07 multiple techniques with depth alpha or rt
// ______________________________________________________________
#if OPENGL
#define SV_POSITION POSITION
#define VS_SHADERMODEL vs_3_0
#define PS_SHADERMODEL ps_3_0
#else
#define VS_SHADERMODEL vs_4_0_level_9_1
#define PS_SHADERMODEL ps_4_0_level_9_1
#endif

//_______________________________________________________________
// common
//_______________________________________________________________
bool     displayVertexNormalShading;

float    alphaDiscardThreshold;
float3   lightDir;
float4   lightColor;
float    ambientColor;
float4x4 gworld;
//float4x4 gview;
float4x4 gworldviewprojection;

Texture2D TextureA;
sampler2D TextureSamplerA = sampler_state
{
    /*AddressU = Wrap;
    AddressV = Wrap;
    MinFilter = Anisotropic;
    MagFilter = Anisotropic;*/
    //MipFilter = Point;
    Texture = <TextureA>;
};

//_______________________________________________________________
// structs TechniqueA
//_______________________________________________________________
struct VertexShaderInput
{
    float4 Position : POSITION0;
    float4 Color : COLOR0;
    float2 TexureCoordinateA : TEXCOORD0;
};
struct VertexShaderOutput
{
    float4 Position : SV_Position;
    float4 Color : COLOR0;
    float2 TexureCoordinateA : TEXCOORD0;
    float2 Depth : TEXCOORD1;
};
struct PixelShaderOutput
{
    float4 Color : COLOR0;
};
//______________________________________________________________
// shader techniques TechniqueA
//______________________________________________________________
VertexShaderOutput VertexShaderFunctionA(VertexShaderInput input)
{
    VertexShaderOutput output;
    output.Position = mul(input.Position, gworldviewprojection);
    float4 col = input.Color;
    output.Color = col;
    output.TexureCoordinateA = input.TexureCoordinateA;

    output.Depth.x = output.Position.z;
    output.Depth.y = output.Position.w;

    return output;
}

PixelShaderOutput PixelShaderFunctionA(VertexShaderOutput input)
{
    PixelShaderOutput output;
    output.Color = tex2D(TextureSamplerA, input.TexureCoordinateA) * input.Color;

    // example 1
    //float depth = input.Depth.x / input.Depth.y;
    //output.Color = (float4(depth, depth, depth, 0)); //Draw the depth in gray

    // example 2
    float depth = input.Depth.x / input.Depth.y;
    if (output.Color.a < alphaDiscardThreshold) // .02f if (output.Color.a < AlphaTestValue)
        clip(-1); // clip(1);

    return output;
}

technique TechniqueA
{
    pass
    {
        VertexShader = compile VS_SHADERMODEL VertexShaderFunctionA();
        PixelShader = compile PS_SHADERMODEL PixelShaderFunctionA();
    }
}

//_______________________________________________________________
// structs TechniqueB
//_______________________________________________________________
struct VertexShaderInputB
{
    float4 Position : POSITION0;
    float3 Normal : NORMAL0;
    float4 Color : COLOR0;
    float2 TexureCoordinateA : TEXCOORD0;
};
struct VertexShaderOutputB
{
    float4 Position : SV_Position;
    float3 Normal : NORMAL0;
    float4 Color : COLOR0;
    float2 TexureCoordinateA : TEXCOORD0;
};
struct PixelShaderOutputB
{
    float4 Color : COLOR0;
};
//______________________________________________________________
// shader techniques TechniqueB
//______________________________________________________________
VertexShaderOutputB VertexShaderFunctionB(VertexShaderInputB input)
{
    VertexShaderOutputB output;
    output.Position = mul(input.Position, gworldviewprojection);
    output.TexureCoordinateA = input.TexureCoordinateA * input.Color;
    float3 normal = normalize(mul(input.Normal, gworld));
    output.Normal = normal;
    float4 col = input.Color;
    float alpha = col.a;
    float lightIntensity = saturate(dot(normal, lightDir));
    col = ((col *.5 + lightColor *.5) * (lightIntensity * (1 - ambientColor))) + (col * ambientColor);
    col.a = alpha;
    output.Color = col;
    return output;
}

PixelShaderOutputB PixelShaderFunctionB(VertexShaderOutputB input)
{
    PixelShaderOutputB output;
    output.Color = tex2D(TextureSamplerA, input.TexureCoordinateA) * input.Color;
    // just display vertex normal shading
    if (displayVertexNormalShading)
    {
        output.Color = input.Color;
    }
    return output;
}

technique TechniqueB
{
    pass
    {
        VertexShader = compile VS_SHADERMODEL VertexShaderFunctionB();
        PixelShader = compile PS_SHADERMODEL PixelShaderFunctionB();
    }
}

Looks like vertexlighting in the screenshots. You are doing NdotL in vertexshader instead of pixel if you wanted pixel lighting it should be in the ps. i also did not see the dot product.
Or i did not understand what you want to achieve which is highly possible :astonished:

When i pick it up again ill try to move it to the pixel shader see what happens.
All its doing is changing the color intensity at each vertices from white towards black if the dot is less then 1 then the color should get lerped as its passed to the shader right ?
i mean it clearly is just not on certain edges. which is really weird.

In the case not used here the lightcolor is just multiplied against the texture as a intensity its super hard to see the lines with a texture drawn.

It really does look like a normal is off on just one vertice of every other quad i must of messed something up somewhere…

The white line in the image i circled that’s the weird thing.

In the basic effect one its even more pronounced.

The line goes from white to black but stays white nearly the entire way.
That’s riding on the edge of a triangle, from a white vertice to a black one.
Really weird

I’ve finally found your dot product, easier to look at the code when not on my mobile ^^

If you try the “shaded” mode on any model in FBX review app from autodesk, sometimes it gives about the same result. It may come from the normals/tangents missing or binormals bi tangents. (the GenerateTangentFrames checkbox in MG pipeline tool is related to this too)

I think if you do the dot product in the vertex shader it is done per vertex whereas in the pixelshader it is per pixel (if you have a normal to sample from, if not i don’t know as I’ve never tried).
Do you use a normal texture to sample from ?

I generate them after creating the mesh vertices with the below.

Maybe i messed it up, It calculates the normal from all connected triangles to the vertice by averaging them…
I tried uping the precision just to make sure some weird precision loss thing wasn’t going on and even normalized them in the function again.
But i can’t tell if they are correct by eye. ill have to make some separate test to see if they are coming out wrong. Don’t have time for it today.

        /// <summary>
        /// This method creates smooth normals from a quad structured vertice mesh array
        /// </summary>
        private PositionNormalColorUv[] CreateSmoothNormals(PositionNormalColorUv[] vertices, int[] indexs)
        {
            // Under quads there are two triangles per quad 
            // such that the triangle associated with normals in a proper indexed vertice list
            // will describe per triangle all shared normals for each vertice.
            // hence each vertice will be shared by 6 triangles ranging from 0 to 5
            // so that for each vertice we must must calculate the surrounding triangle normals to average them.
            // well just brute force this as its only done once on creation.

            int tvertmultiplier = 3;
            int triangles = (int)(indexs.Length / tvertmultiplier);

            // Loop the vertices
            for (int currentTestedVerticeIndex = 0; currentTestedVerticeIndex < vertices.Length; currentTestedVerticeIndex++)
            {
                // Reset are sum and total counts
                float total = 0;
                Vector3 sum = Vector3.Zero;

                // loop thru each triangle
                for (int t = 0; t < triangles; t++)
                {
                    int tvstart = t * tvertmultiplier;
                    int tindex0 = tvstart + 0;
                    int tindex1 = tvstart + 1;
                    int tindex2 = tvstart + 2;
                    var vindex0 = indices[tindex0];
                    var vindex1 = indices[tindex1];
                    var vindex2 = indices[tindex2];
                    // Test each triangle for the currentTestedVertice presence
                    if (vindex0 == currentTestedVerticeIndex || vindex1 == currentTestedVerticeIndex || vindex2 == currentTestedVerticeIndex)
                    {
                        // Obtain a orthagonal direction from the surface of this triangle. 
                        // This is basically a non unit length normal.
                        // Multiply by some amount 10 here to increase precision and avoid nans.
                        var n0 = (vertices[vindex1].Position - vertices[vindex0].Position) * 100f;
                        var n1 = (vertices[vindex2].Position - vertices[vindex1].Position) * 100f;
                        n0.Normalize();
                        n1.Normalize();
                        // Sum the cross product and increment the total
                        var cnorm = Vector3.Cross(n0, n1);
                        sum += cnorm * 100f;
                        total += 1;
                    }
                }
                // Find the average 
                if (total > 0)
                {
                    var averagednormal = sum / total;
                    // Normalize the result 
                    averagednormal.Normalize();
                    // Assign the normal to the vertice
                    vertices[currentTestedVerticeIndex].Normal = averagednormal;
                    if (showSmoothing)
                    {
                        Console.WriteLine(" SmoothedNormals: vertices[" + currentTestedVerticeIndex + "].Position:  " + vertices[currentTestedVerticeIndex].Position);
                        Console.WriteLine(" SmoothedNormals: vertices[" + currentTestedVerticeIndex + "].Normal:  " + averagednormal);
                    }
                }
            }
            return vertices;
        }