SSAO shader problem

Hi everyone,

I’ve been working on getting SSAO in to my game, but I have been stuck on a problem.
I’m sure its going to be something dumb, but any assistance would be great, as i’ve played with it for half a day now with little luck.

This is what I’m getting:

This is straight front in World Space, default camera, just moved it back from the (0,0,0).

This is the same camera position just turned about ~70 degrees for display reasons. I could fly around and as long as I look towards world z its fine, everywhere else its between working and broken.

Here is the shader I’m using. The HBO for loop is from Kosmonauthgames shaders, the rest is build from “scratch” to try and get it to work in my engine.

float3 cameraPosition;

float3 cornerFustrum;

matrix View;
matrix InverseViewProjection;
matrix Projection;

Texture2D NormalMap;
Texture2D DepthMap;
texture2D RandomMap;

int Samples = 8;

float Strength = 4;

float SampleRadius = 0.5f;//0.05 - 0.5

float2 Resolution = float2(3840, 2160);

SamplerState texSampler
{
	AddressU = CLAMP;
	AddressV = CLAMP;
	MagFilter = POINT;
	MinFilter = POINT;
	Mipfilter = POINT;
};

sampler RandomSampler = sampler_state
{
	Texture = (RandomMap);
	MipFilter = Linear;
	MinFilter = Linear;
	MagFilter = Linear;
};

struct VertexInput
{
	float4 Position : POSITION0;
	float2 TexCoord : TEXCOORD0;
};

struct PixelInput
{
	float4 Position : POSITION0;
	float2 TexCoord : TEXCOORD0;
};

PixelInput SSAOVertexShader(VertexInput input)
{
	PixelInput pi = (PixelInput)0;

	pi.Position = input.Position;
	pi.TexCoord = input.TexCoord;

	return pi;
}


float linearizeDepth(in float depth) {
	float n = 0.1;
	float f = 500.0f;
	//return n / (f - depth * (f - n)) * f;

	float z = depth * 2.0 - 1.0; // back to NDC 
	return (2.0 * n * f) / (f + n - z * (f - n));

}


float3 getPosition(in float2 uv)
{ 
	float depth = DepthMap.SampleLevel(texSampler, uv, 0).r;
	//depth = linearizeDepth(depth);
	// Convert position to world space
	float4 position;

	position.xy =  uv.xy * 2.0f - 1.0f;
	position.y = -position.y;
	position.z = depth;
	position.w = 1.0f;

	position = mul(position, InverseViewProjection);
	position /= position.w;		
	
	// Convert world space to view space
	position = mul(position, View);
		
	return position.xyz;
}



float3 randomNormal(float2 tex)
{
	float noiseX = (frac(sin(dot(tex, float2(15.8989f, 76.132f) * 1.0f)) * 46336.23745f));
	float noiseY = (frac(sin(dot(tex, float2(11.9899f, 62.223f) * 2.0f)) * 34748.34744f));
	float noiseZ = (frac(sin(dot(tex, float2(13.3238f, 63.122f) * 3.0f)) * 59998.47362f));
	return normalize(float3(noiseX, noiseY, noiseZ));
}

float weightFunction(float3 vec3, float radius)
{
	// NVIDIA's weighting function
	return 1.0 - /*length(vec3) / radius;*/pow(length(vec3) / radius, 2.0);
}





float4 SSAOPixelShader(PixelInput input) : COLOR0
{

	const float3 kernel[] =
{
	float3(0.2024537f, 0.841204f, -0.9060141f),
	float3(-0.2200423f, 0.6282339f, -0.8275437f),
	float3(-0.7578573f, -0.5583301f, 0.2347527f),
	float3(-0.4540417f, -0.252365f, 0.0694318f),
	float3(0.3677659f, 0.1086345f, -0.4466777f),
	float3(0.8775856f, 0.4617546f, -0.6427765f),
	float3(-0.8433938f, 0.1451271f, 0.2202872f),
	float3(-0.4037157f, -0.8263387f, 0.4698132f),
	float3(0.7867433f, -0.141479f, -0.1567597f),
	float3(0.4839356f, -0.8253108f, -0.1563844f),
	float3(0.4401554f, -0.4228428f, -0.3300118f),
	float3(0.0019193f, -0.8048455f, 0.0726584f),
	float3(-0.0483353f, -0.2527294f, 0.5924745f),
	float3(-0.4192392f, 0.2084218f, -0.3672943f),
	float3(-0.6657394f, 0.6298575f, 0.6342437f),
	float3(-0.0001783f, 0.2834622f, 0.8343929f),
};

float2 texCoord = float2(input.TexCoord);

//get normal data from the NormalMap
float4 normalData = NormalMap.Sample(texSampler, texCoord);
//tranform normal back into [-1,1] range
float3 currentNormal = 2.0f * normalData.xyz - 1.0f;

float linearDepth = DepthMap.Sample(texSampler, texCoord).r;
//linearDepth = linearizeDepth(linearDepth);



if (linearDepth < 0.0000001f)
{
	return float4(1, 1, 1, 1);
}


float3 currentPos = getPosition(texCoord);//Position in VS

float currentDistance = -currentPos.z;

float2 aspectRatio = float2(min(1.0f, Resolution.y / Resolution.x), min(1.0f, Resolution.x / Resolution.y));

float amount = 1.0f;

float3 noise = randomNormal(texCoord);

//HBAO 2 dir
int sampleshalf = Samples * 0.5;
for (int i = 0; i < sampleshalf; i++)
{
	float3 kernelVec = reflect(kernel[i], noise);
	kernelVec.xy *= aspectRatio;

	float radius = SampleRadius;

	kernelVec.xy = (kernelVec.xy / currentDistance) * radius;

	float biggestAnglePos = 0.0f;

	float biggestAngleNeg = 0.0f;

	float wAO = 0.0;

	for (int b = 1; b <= 4; b++)
	{
		float3 sampleVec = getPosition(texCoord + kernelVec.xy * b / 4.0f) - currentPos;

		float sampleAngle = dot(normalize(sampleVec), currentNormal);

		sampleAngle *= step(0.3, sampleAngle);

		if (sampleAngle > biggestAnglePos)
		{
			wAO += saturate(weightFunction(sampleVec, radius) * (sampleAngle - biggestAnglePos));

			biggestAnglePos = sampleAngle;
		}

		sampleVec = getPosition(texCoord - kernelVec.xy * b / 4.0f) - currentPos;

		sampleAngle = dot(normalize(sampleVec), currentNormal);

		if (sampleAngle > biggestAngleNeg)
		{
			wAO += saturate(weightFunction(sampleVec, radius) * (sampleAngle - biggestAngleNeg));

			biggestAngleNeg = sampleAngle;
		}
	}
		

	amount -= wAO / Samples * Strength;
}

return float4(amount, amount, amount, amount);
}

technique SSAO
{
	pass Pass0
	{
		VertexShader = compile vs_4_0 SSAOVertexShader();
		PixelShader = compile ps_4_0 SSAOPixelShader();
	}
}

At this point I’m reasonably sure its something to do with my camera view direction as everyone else seam to be using ViewRay’s left right and center.
Thanks for anyone that can help or guide me in the correct direction, pun not intended, but I feel like its going to end up being ironic.

Thanks for everyone that had a look.
I think I may have solved my problem, probably not the most elegant way of doing it, but the most simplistic out of all the examples I have seen (at least for me).
Here is the final code for those who may care, may come back to it if I see any strangeness with it.

matrix View;
matrix InverseProjection;

Texture2D NormalMap;
Texture2D DepthMap;

int Samples = 8;

float Strength = 4;

float SampleRadius = 0.5f;//0.05 - 0.5

float2 Resolution = float2(3840, 2160);

SamplerState texSampler
{
	AddressU = CLAMP;
	AddressV = CLAMP;
	MagFilter = POINT;
	MinFilter = POINT;
	Mipfilter = POINT;
};

struct VertexInput
{
	float4 Position : POSITION0;
	float2 TexCoord : TEXCOORD0;
};

struct PixelInput
{
	float4 Position : POSITION0;
	float2 TexCoord : TEXCOORD0;
};

PixelInput SSAOVertexShader(VertexInput input)
{
	PixelInput pi = (PixelInput)0;

	pi.Position = input.Position;
	pi.TexCoord = input.TexCoord;

	return pi;
}



float4 getPosition(in float2 uv)
{ 
	float depth = DepthMap.SampleLevel(texSampler, uv, 0).r;
	

	//compute screen-space position	
	float4 position;
	position.xy =  uv.xy * 2.0f - 1.0f;
	position.y = -position.y;
	position.z = depth;
	position.w = 1.0f;
	
	position = mul(position, InverseProjection);
	position.xyz /= position.w;	
			
	return position;
}



float3 randomNormal(float2 tex)
{
	float noiseX = (frac(sin(dot(tex, float2(15.8989f, 76.132f) * 1.0f)) * 46336.23745f));
	float noiseY = (frac(sin(dot(tex, float2(11.9899f, 62.223f) * 2.0f)) * 34748.34744f));
	float noiseZ = (frac(sin(dot(tex, float2(13.3238f, 63.122f) * 3.0f)) * 59998.47362f));
	return normalize(float3(noiseX, noiseY, noiseZ));
}

float weightFunction(float3 vec3, float radius)
{
	// NVIDIA's weighting function
	return 1.0 - pow(length(vec3) / radius, 2.0);
}



float4 SSAOPixelShader(PixelInput input) : COLOR0
{

	const float3 kernel[] =
{
	float3(0.2024537f, 0.841204f, -0.9060141f),
	float3(-0.2200423f, 0.6282339f, -0.8275437f),
	float3(-0.7578573f, -0.5583301f, 0.2347527f),
	float3(-0.4540417f, -0.252365f, 0.0694318f),
	float3(0.3677659f, 0.1086345f, -0.4466777f),
	float3(0.8775856f, 0.4617546f, -0.6427765f),
	float3(-0.8433938f, 0.1451271f, 0.2202872f),
	float3(-0.4037157f, -0.8263387f, 0.4698132f),
	float3(0.7867433f, -0.141479f, -0.1567597f),
	float3(0.4839356f, -0.8253108f, -0.1563844f),
	float3(0.4401554f, -0.4228428f, -0.3300118f),
	float3(0.0019193f, -0.8048455f, 0.0726584f),
	float3(-0.0483353f, -0.2527294f, 0.5924745f),
	float3(-0.4192392f, 0.2084218f, -0.3672943f),
	float3(-0.6657394f, 0.6298575f, 0.6342437f),
	float3(-0.0001783f, 0.2834622f, 0.8343929f),
};

float2 texCoord = float2(input.TexCoord);

//get normal data from the NormalMap
float4 normalData = NormalMap.Sample(texSampler, texCoord);


//tranform normal back into [-1,1] range
float3 currentNormal = 2.0f * normalData.xyz - 1.0f;

//transform
currentNormal = normalize(mul(currentNormal, View));

float linearDepth = DepthMap.Sample(texSampler, texCoord).r;


if (linearDepth < 0.00000001f)
{
	return float4(1, 1, 1, 1);
}


float3 currentPos = getPosition(texCoord);//Position in VS

float currentDistance = -currentPos.z;

float2 aspectRatio = float2(min(1.0f, Resolution.y / Resolution.x), min(1.0f, Resolution.x / Resolution.y));

float amount = 1.0f;

float3 noise = randomNormal(texCoord);

//HBAO 2 dir
int sampleshalf = Samples * 0.5;
for (int i = 0; i < sampleshalf; i++)
{
	float3 kernelVec = reflect(kernel[i], noise);
	kernelVec.xy *= aspectRatio;

	float radius = SampleRadius;

	kernelVec.xy = (kernelVec.xy / currentDistance) * radius;

	float biggestAnglePos = 0.0f;

	float biggestAngleNeg = 0.0f;

	float wAO = 0.0;

	for (int b = 1; b <= 4; b++)
	{
		float3 sampleVec = getPosition(texCoord + kernelVec.xy * b / 4.0f) - currentPos;

		float sampleAngle = dot(normalize(sampleVec), currentNormal);

		sampleAngle *= step(0.3, sampleAngle);

		if (sampleAngle > biggestAnglePos)
		{
			wAO += saturate(weightFunction(sampleVec, radius) * (sampleAngle - biggestAnglePos));

			biggestAnglePos = sampleAngle;
		}

		sampleVec = getPosition(texCoord - kernelVec.xy * b / 4.0f) - currentPos;

		sampleAngle = dot(normalize(sampleVec), currentNormal);

		if (sampleAngle > biggestAngleNeg)
		{
			wAO += saturate(weightFunction(sampleVec, radius) * (sampleAngle - biggestAngleNeg));

			biggestAngleNeg = sampleAngle;
		}
	}
		

	amount -= wAO / Samples * Strength;
}

return float4(amount, amount, amount, amount);
}

technique SSAO
{
	pass Pass0
	{
		VertexShader = compile vs_4_0 SSAOVertexShader();
		PixelShader = compile ps_4_0 SSAOPixelShader();
	}
}
2 Likes

What was the fix? I didn’t spot anything in your code (was a bit tough to read, with the on/off formatting).

I see he’s only multiplying by inverse projection now instead of inverse view projection and then again by view. The dividing by w was only done in the middle instead of again at the end before, so I wonder if it’s working now that the division by w comes after all transformations.