The 1 you specify as an offset basically is the whole width/height of the image because as far as the shader is concerned the image has the size {1, 1} (so {.5, .5} would be the exact center…
So you’d have to divide by texturesize in order to get the neighboring pixels…
(I pass the texturesize to the shader… maybe there’s another way to get it, but I don’t know)
This code here works for me and has a kernel-size of 5 instead of 3 like what you tried to do. That actually makes the outline broader…
sampler ColorMapSampler : register(s0);
float4 MaxColor;
float4 BoundaryColor;
float4 BoundarySecondColor;
float2 TextureSize;
float2 TexelSize; // should be (1f/textureSize.X, 1f/textureSize.Y)
float4 PixelShaderColorLimit(float4 pos : SV_POSITION, float4 color1 : COLOR0, float2 Tex : TEXCOORD0) : SV_TARGET0
{
float4 sc = tex2D(ColorMapSampler, Tex);
if(sc.a > MaxColor.a) {
return MaxColor;
}
return sc;
}
float4 PixelShaderBoundaries(float4 pos : SV_POSITION, float4 color1 : COLOR0, float2 Tex : TEXCOORD0) : SV_TARGET0
{
// Determine the floating point size of a texel for a screen with this specific width and height,
// Since the TEXCOORD0 are in the form of (0f-1f, 0f-1f).
// Is passed since this shader would use 66 arithmetic instructions otherwise in an older, unoptimized version (out of 64 available in shader model 2.0).
//float2 TexelSize = float2(1.0f / TextureSize.x, 1.0f / TextureSize.y);
// X = current pixel, F = first neighbours, O = second neighbours.
// O
// O F O
// O F X F O
// O F O
// O
// We're reading the points into 3 vectors in order to use vector-operations (saves arithmetic operations).
// The next line demonstrates how to set a result-vector to 1 if the point is not blank and zero, if it is.
// r1 = (first > 0.0f) ? float4(1.0f, 1.0f, 1.0f, 1.0f) : float4(0.0f, 0.0f, 0.0f, 0.0f);
// The next line sums up the calculated vector using a floating-point-vector-operation. The result is r1.x+r1.y+r1.z+r1.w
// result = dot(r1, 1);
// The next if just tests if at least one is blank and at least one is not blank.
float4 Color = tex2D(ColorMapSampler, Tex);
float4 first;
first.x = tex2D(ColorMapSampler, Tex + float2(0.0f,-TexelSize.y)).a;
first.y = tex2D(ColorMapSampler, Tex + float2(0.0f,TexelSize.y)).a;
first.z = tex2D(ColorMapSampler, Tex + float2(-TexelSize.x,0.0f)).a;
first.w = tex2D(ColorMapSampler, Tex + float2(TexelSize.x,0.0f)).a;
float4 second1;
second1.x = tex2D(ColorMapSampler, Tex + float2(0.0f,-2.0f * TexelSize.y)).a;
second1.y = tex2D(ColorMapSampler, Tex + float2(TexelSize.x,-TexelSize.y)).a;
second1.z = tex2D(ColorMapSampler, Tex + float2(2.0f * TexelSize.x,0.0f)).a;
second1.w = tex2D(ColorMapSampler, Tex + float2(TexelSize.x,TexelSize.y)).a;
float4 second2;
second2.x = tex2D(ColorMapSampler, Tex + float2(0.0f,2.0f * TexelSize.y)).a;
second2.y = tex2D(ColorMapSampler, Tex + float2(-TexelSize.x,-TexelSize.y)).a;
second2.z = tex2D(ColorMapSampler, Tex + float2(-2.0f * TexelSize.x,0.0f)).a;
second2.w = tex2D(ColorMapSampler, Tex + float2(-TexelSize.x,-TexelSize.y)).a;
float4 r1;
float4 r2;
float result;
r1 = (first > 0.0f) ? float4(1.0f, 1.0f, 1.0f, 1.0f) : float4(0.0f, 0.0f, 0.0f, 0.0f);
result = dot(r1, 1);
if(result > 0.0f && result < 4.0f) {
return BoundaryColor;
}
r1 = (second1 > 0.0f) ? float4(1.0f, 1.0f, 1.0f, 1.0f) : float4(0.0f, 0.0f, 0.0f, 0.0f);
r2 = (second2 > 0.0f) ? float4(1.0f, 1.0f, 1.0f, 1.0f) : float4(0.0f, 0.0f, 0.0f, 0.0f);
result = dot(r1, 1) + dot(r2, 1);
if(result > 0.0f && result < 8.0f) {
return BoundarySecondColor;
}
return Color;
}
technique ColorLimit
{
pass P0
{
#if SM4
PixelShader = compile ps_4_0_level_9_1 PixelShaderColorLimit();
#elif SM3
PixelShader = compile ps_3_0 PixelShaderColorLimit();
#else
PixelShader = compile ps_2_0 PixelShaderColorLimit();
#endif;
}
}
technique Boundaries
{
pass P0
{
#if SM4
PixelShader = compile ps_4_0_level_9_1 PixelShaderBoundaries();
#elif SM3
PixelShader = compile ps_3_0 PixelShaderBoundaries();
#else
PixelShader = compile ps_2_0 PixelShaderBoundaries();
#endif;
}
}
Edit: Now that I actually read my code again I wanna say: I really, wholeheartedly apologize for the rest of the code… It’s been years and I was young and I needed the money (and since this part worked, adhering to ancient coding traditions, I never touched it again) 
