Looking for SSAO guidance

Hello!

I am fairly new to shaders, but I’ve been making progress. I have a simple set of shaders that I’m trying to implement, but I’m having a very weird outcome. My process is like so.

  1. Render Scene to RenderTarget
  2. Render Depth to RenderTarget using Shader.
  3. Set Depth & Random Normal textures to SSAO shader and render to RenderTarget using fullscreen quad (I have no idea if this is right, this is what I’ve observed that many others have been doing),
  4. Combine the Scene Render Target from Step 1 with the SSAO Render Target from Step 3 and Render Fullscreen Quad.

My output is a giant grey screen. In theory, are the high level steps in line with what one would need to do to apply SSAO to a scene? I’ve included various screenshots and files. I’m a bit lost and I’m thinking I may need to back off of trying to implement this until I understand this better. I’m open to trying anything at this point.

Scene RenderTarget

Depth Render Target

Step 2: Depth Shader

float4x4 WorldView;
float4x4 ITWorldView;
float4x4 WorldViewProjection;

float FarClip;

struct VS_INPUT
{
float4 Position: POSITION0;
float3 Normal : NORMAL;
};

struct VS_OUTPUT
{
float4 Position: POSITION0;
float3 Normal : TEXCOORD0;
float4 vPositionVS : TEXCOORD1;
};

VS_OUTPUT DepthVertexShaderFunction(VS_INPUT IN)
{
VS_OUTPUT Output;

Output.Position = mul(IN.Position, WorldViewProjection);
Output.vPositionVS = mul(IN.Position, WorldView);
Output.Normal = mul(IN.Normal, ITWorldView);

return Output;
}

float4 DepthPixelShaderFunction(VS_OUTPUT IN) : COLOR0
{
float depth = IN.vPositionVS.z / FarClip;
IN.Normal = normalize(IN.Normal);
return float4(IN.Normal.x, IN.Normal.y, IN.Normal.z, depth);
}

technique Depth
{
pass Pass1
{

    VertexShader = compile vs_4_0 DepthVertexShaderFunction();
    PixelShader = compile ps_4_0 DepthPixelShaderFunction();
}

}

Step 3: SSAO Shader

float sampleRadius;
float distanceScale;
float4x4 Projection;

float3 cornerFustrum;

struct VS_OUTPUT
{
float4 pos : POSITION;
float2 TexCoord : TEXCOORD0;
float3 viewDirection : TEXCOORD1;
};

VS_OUTPUT VS(
float4 Position : POSITION, float2 TexCoord : TEXCOORD0)
{
VS_OUTPUT Out = (VS_OUTPUT)0;

Out.pos = Position;
Position.xy = sign(Position.xy);
Out.TexCoord = (float2(Position.x, -Position.y) + float2( 1.0f, 1.0f ) ) * 0.5f;
float3 corner = float3(-cornerFustrum.x * Position.x,
  	cornerFustrum.y * Position.y, cornerFustrum.z);

Out.viewDirection = corner;

return Out;

}

texture depthTexture;
texture randomTexture;

sampler2D depthSampler = sampler_state
{
Texture = ;
ADDRESSU = CLAMP;
ADDRESSV = CLAMP;
MAGFILTER = LINEAR;
MINFILTER = LINEAR;
};

sampler2D RandNormal = sampler_state
{
Texture = ;
ADDRESSU = WRAP;
ADDRESSV = WRAP;
MAGFILTER = LINEAR;
MINFILTER = LINEAR;
};

float4 PS(VS_OUTPUT IN) : COLOR0
{
float4 samples[16] =
{
float4(0.355512, -0.709318, -0.102371, 0.0 ),
float4(0.534186, 0.71511, -0.115167, 0.0 ),
float4(-0.87866, 0.157139, -0.115167, 0.0 ),
float4(0.140679, -0.475516, -0.0639818, 0.0 ),
float4(-0.0796121, 0.158842, -0.677075, 0.0 ),
float4(-0.0759516, -0.101676, -0.483625, 0.0 ),
float4(0.12493, -0.0223423, -0.483625, 0.0 ),
float4(-0.0720074, 0.243395, -0.967251, 0.0 ),
float4(-0.207641, 0.414286, 0.187755, 0.0 ),
float4(-0.277332, -0.371262, 0.187755, 0.0 ),
float4(0.63864, -0.114214, 0.262857, 0.0 ),
float4(-0.184051, 0.622119, 0.262857, 0.0 ),
float4(0.110007, -0.219486, 0.435574, 0.0 ),
float4(0.235085, 0.314707, 0.696918, 0.0 ),
float4(-0.290012, 0.0518654, 0.522688, 0.0 ),
float4(0.0975089, -0.329594, 0.609803, 0.0 )
};

IN.TexCoord.x += 1.0/1600.0;
IN.TexCoord.y += 1.0/1200.0;

normalize (IN.viewDirection);
float depth = tex2D(depthSampler, IN.TexCoord).a;
float3 se = depth * IN.viewDirection;

float3 randNormal = tex2D( RandNormal, IN.TexCoord * 200.0 ).rgb;

float3 normal = tex2D(depthSampler, IN.TexCoord).rgb;
float finalColor = 0.0f;

for (int i = 0; i < 16; i++)
{
float3 ray = reflect(samples[i].xyz,randNormal) * sampleRadius;

  //if (dot(ray, normal) < 0)
  //	ray += normal * sampleRadius;
  	
  float4 sample = float4(se + ray, 1.0f);
  float4 ss = mul(sample, Projection);
  float2 sampleTexCoord = 0.5f * ss.xy/ss.w + float2(0.5f, 0.5f);
  
  sampleTexCoord.x += 1.0/1600.0;
  sampleTexCoord.y += 1.0/1200.0;
  float sampleDepth = tex2D(depthSampler, sampleTexCoord).a;
  
  if (sampleDepth == 1.0)
  {
  	finalColor ++;
  }
  else
  {		
  	float occlusion = distanceScale* max(sampleDepth - depth, 0.0f);
  	finalColor += 1.0f / (1.0f + occlusion * occlusion * 0.1);
  }

}

return float4(finalColor/16, finalColor/16, finalColor/16, 1.0f);
}

technique SSAO
{
pass P0
{
VertexShader = compile vs_4_0 VS();
PixelShader = compile ps_4_0 PS();
}
}

Step 4: SSAO Combine

//sampler baseSampler : register(s0);
texture SSAOTex;

sampler2D SSAOSampler = sampler_state
{
Texture = ;
ADDRESSU = CLAMP;
ADDRESSV = CLAMP;
MAGFILTER = LINEAR;
MINFILTER = LINEAR;
};

texture SceneTexture;

sampler2D baseSampler = sampler_state
{
Texture = ;
ADDRESSU = CLAMP;
ADDRESSV = CLAMP;
MAGFILTER = LINEAR;
MINFILTER = LINEAR;
};

float4 PixelShaderFunction(float2 TexCoord :TEXCOORD0) : COLOR0
{
TexCoord.x += 1.0/1600.0;
TexCoord.y += 1.0/1200.0;
return tex2D( SSAOSampler, TexCoord ).r * (tex2D(baseSampler,TexCoord) );
}

technique Merge
{
pass Pass1
{
PixelShader = compile ps_4_0 PixelShaderFunction();
}
}

Im on mobile right now - can you post a screen of the ssao rendertarget only?

In theory the way you do it is correct, I’ve looked briefly over it and it seems alright. Where did you take the shader from?

One issue might be the depthbuffer generation, if that’s not setup the right way results may be lacking. But I suppose you have both shaders from the same source?

Oh and another thing - the 1600 and 1200 part is not needed any more.
That was an XNA issue where you had to shift half a pixel for texture coordinates. I guess the original Shader ran in 800x600 then :wink:

Is the second screenshot the depth rendertarger ?

Yes. It looks like depth is in the alpha and the surface normal is in RGB.

Ok but when I open it in PS, there is no alpha channel. I was wondering how it could be ok without being able to catch it.

Posting images inline in the forum strips the alpha channel.

Kosmo,

You are correct, the depth, ssao, and ssao combine shaders came from the same place. I’m hoping to extract the “how” so that I never need to borrow again, lol.

This is what my SSAO target and final combine render look like. One observation. If I walk around while rendering the final combined, the shades of gray change rapidly.

As always, thank for your guidance!

**SSAO Target (Step 3) Yes, it actually is all white, the forums aren’t bugged :slight_smile: **

SSAO Combined (Step 4)

This might be a guess, but try adding a POSITION parameter to the pixel shader input.

1 Like

Going to give that a go now. I did discover a very small bug in my depth pass, where I wasn’t incline the mode bone transforms as part of the world matrix. My depth target is looking better. It didn’t fix anything, but it’s one more thing to cross off the list.

As soon as I try the position change, I’ll report back.

Sorry, to which shader were you thinking? The final pass shader?

Yes, the final pass shader that only has a pixel shader.

No luck, but I am noticing something new. Earlier, I spoke about how when I’d move through the world, I’d get random shades of gray where the screen color would change. Now, since adding that parameter, it is incredibly rapid and more varying.

EDIT: I’m starting to flat out suspect that the shader/shaders is just bad. I may have to go to whitepapers to implement one from the ground up.

WOOHOO, I’M GETTING SOMEWHERE!!!

I ended up adding this line in my PixelShader during the SSAO pass:

IN.TexCoord += halfPixel;

Now when I render my SSAO render target, I get the following in one of the hallways! I’ve discerned that the SSAO Combine shader may be the thing causing the random shades of gray.

EDIT: I’m stopping for the day, but I’ve made progress. At this point I’ve got my scene render target and a successful ssao render target. My final pass combine shader flat out doesn’t work. I’m going to keep going to see what I can find, but I’m stumped on how to get the two render targets combined. Thanks to everyone for giving me advice, I always learn a ton in this forum. If anyone has any thoughts or ideas, I’m open to try anything.

1 Like

Can someone point me to how to correctly get linearDepth and consequently the raymarching in viewspace with linear depth?

I finally got my combine shader working for step 4. It finally freaking works. As I said back when I got my point lighting working, I’m going to blog a tutorial specifically for monogame about setting up point lights with SSAO. This is truly an awesome community.

Before SSAO

After SSO

P.S. Kosmo, Since you’ve given me so much help, I will return the favor. I’m going to sit down tonight and research linearDepth with raymarching tonight. I can’t promise that I’ll be of much help, but I’ll give it my best shot!

1 Like

Good to see. What was the issue with the shader?

I’m not sure technically what it was because the approaches seemed so different, but I ultimately ended up adding a VertexShader to the combine shader just to return the basic data. For the PixelShader, the tex2d call now utilizes .UV from a full input structure instead of just a raw TexCoord. The two sampled pieces of data were then multiplied.

I’d post the shader but I’m on mobile now. I will definitely be sharing this all back to the community!

Thank you for all your help!

Hello,

Great job, it looks great! Have you plans to share the code behind it :slight_smile: ?

THanks!

You can check this out, might help you: There are specific SSAO shaders and code structures.

Edit: the SSAO used there is HBAO but i thought it was a better documented version.
If you search for ssao xna on the internet you should find good resources though

Thank you! I’ll take a look at that. I already use you great bloom and color grading effects :wink: