I have a simple ‘transition’ shader that returns either the pixel from texture A or texture B depending on the state of the M texture. See below.
When using DirectX both versions/attempts work fine. But in OpenGL instead of returning texture B it seems to return M. Making the result is either A or M instead of A or B.
I am not very good with shaders, and obviously there is some difference in OpenGL I am unaware of. Any help would be appreciated.
//_______________________________
// TransitionShader .fx
//
#if SM4
	#define PS_PROFILE ps_4_0
	#define VS_PROFILE vs_4_0
#else
	#define PS_PROFILE ps_3_0
	#define VS_PROFILE vs_3_0
#endif
// the textures
Texture2D TextureA;
sampler2D TextureSamplerA = sampler_state
{
    Texture = <TextureA>;
};
Texture2D TextureB;
sampler2D TextureSamplerB = sampler_state
{
    Texture = <TextureB>;
};
Texture2D TextureMask;
sampler2D TextureSamplerMask = sampler_state
{
    Texture = <TextureMask>;
};
float4 Merge( float4 inPosition : SV_Position,
			    float4 inColor : COLOR0,
			    float2 coords : TEXCOORD0 ) : COLOR0
{
    float4 output;
    float4 M = tex2D(TextureSamplerMask, coords);
    float4 A = tex2D(TextureSamplerA, coords);
    float4 B = tex2D(TextureSamplerB, coords);
    
    // Second Attempt
    if (M.r > 0.5) return A;
    return B;
    // First Attempt 
    //  output.rgb = (B.rgb * M.r) + (A.rgb * (1 - M.r));
    //  output.a = 1;
    //  return output;
}
technique Technique1
{
    pass P1
    {
        PixelShader = compile PS_PROFILE Merge();
    }
}
