[Solved]Render Target Cube Texture Cube How to questions.

This is a continuation of a solved post here [Solved]How would one render the screen to just a portion of a rendertarget ?
It was sort of off topic in the other thread.

I still have some questions on how to render this rendercube to a actual quad cube.
Do i set the light position to be the center of a quad cube and then get the rendered scene texels to do reflection mapping onto a cube ? Id like to also be able to colorize the depth and display it in that way onto a cube for visualization and to make sure it looks proper.

Anyways i got it running i have no idea how bugged up this really is till i get a better scene in there im tired so itll have to wait. But it appears to be doing some shadowing.

Gray is the shadow color in the below picture. So it’s at least partialy working.

Here’s my code so far minus the helper class with the grid.

Game1 and Shader

using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Input;

namespace TestTextureCube
{

    public class Game1 : Game
    {
        GraphicsDeviceManager graphics;
        SpriteBatch spriteBatch;
        // whats the difference here what does the textureCube do exactly
        TextureCube textureCube;
        // i get the rendertargetCube
        RenderTargetCube renderTargetCube;
        //
        Effect effect;
        //
        float near = 1.0f;
        float far = 1000f;
        Matrix world, view, projection;
        Matrix worldGrid, worldCamera, worldLight, worldObj1, worldObj2;
        Vector3 worldGridPosition, worldCameraPosition, worldLightPosition, worldObj1Position, worldObj2Position;

        // something to draw
        Grid3dOrientation grid = new Grid3dOrientation(30, 30, .001f);
        // 2.0 is screen scale with a wvp identity matrix.
        Quad quad = new Quad(1.7f);

        public Game1()
        {
            graphics = new GraphicsDeviceManager(this);
            Content.RootDirectory = "Content";
        }

        protected override void Initialize()
        {
            base.Initialize();
        }

        protected override void LoadContent()
        {
            spriteBatch = this.LoadBasics("Shadow Cube test");
            effect = Content.Load<Effect>("EffectCubeMap");
            textureCube = new TextureCube(GraphicsDevice, 6, false, SurfaceFormat.Single);
            renderTargetCube = new RenderTargetCube(GraphicsDevice, 256, false, SurfaceFormat.Single, DepthFormat.Depth24);
            effect.Parameters["farPlane"].SetValue(far);

            world = Matrix.Identity;
            SetView(world);
            projection = Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * .5f, GraphicsDevice.Viewport.Width / GraphicsDevice.Viewport.Height, near, far);

            worldLightPosition = new Vector3(3f, .1f, -.5f);
            worldLight = CreateWorldToTarget(worldLightPosition, Vector3.Zero, Vector3.Up);
            worldGridPosition = Vector3.Zero;
            worldGrid = Matrix.CreateScale(30f) * Matrix.CreateWorld(Vector3.Zero, Vector3.Forward, Vector3.Up);
            worldCameraPosition = new Vector3(.5f, -.5f, 2f);
            worldCamera = CreateWorldToTarget(worldCameraPosition, Vector3.Zero, Vector3.Up);

        }

        protected override void UnloadContent()
        {
            this.UnloadBasics();
        }

        protected override void Update(GameTime gameTime)
        {
            if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed || Keyboard.GetState().IsKeyDown(Keys.Escape))
                Exit();

            //BasicStuff.Update(gameTime);
            base.Update(gameTime);
        }

        protected override void Draw(GameTime gameTime)
        {
            GraphicsDevice.Clear(Color.Black);
            GraphicsDevice.Clear(ClearOptions.DepthBuffer, Color.Black, 1, 0);
            GraphicsDevice.RasterizerState = new RasterizerState() { FillMode = FillMode.Solid, CullMode = CullMode.None };
            GraphicsDevice.DepthStencilState = new DepthStencilState() { DepthBufferEnable = true, DepthBufferFunction = CompareFunction.LessEqual };

            // ok so lets take a look
            effect.CurrentTechnique = effect.Techniques["CreateLightShadowCubeDepth"];

            // well set a rendertarget to prepare for drawing
            // we set our own view matrix to match at the light shadow location.
            // we draw some stuff to a depth buffer.
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeX);
            SetViewToLight(worldLightPosition, Vector3.Left);
            DrawGridToDepth();

            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeY);
            SetViewToLight(worldLightPosition, Vector3.Down);
            DrawGridToDepth();

            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeZ);
            SetViewToLight(worldLightPosition, Vector3.Forward);
            DrawGridToDepth();

            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveX);
            SetViewToLight(worldLightPosition, Vector3.Right);
            DrawGridToDepth();

            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveY);
            SetViewToLight(worldLightPosition, Vector3.Up);
            DrawGridToDepth();

            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveZ);
            SetViewToLight(worldLightPosition, Vector3.Backward);
            DrawGridToDepth();

            // definately going to need a octree too.

            // Ok so we have our depths in this rendertarget cube.
            // I want to draw with the shadowing.
            // So on this calculation side, im guessing it just does the math and thats the attraction.
            // i suppose i can get data on this thing if i want to see it flattened out.

            // switch to backbuffer
            GraphicsDevice.SetRenderTarget(null);

            // we switch our view now to our actual camera and draw our scene with shadow.... but...
            effect.CurrentTechnique = effect.Techniques["LightShadowCubeBasicShader"];
            effect.Parameters["worldLightPosition"].SetValue(worldLightPosition);
            SetView(worldCamera);
            effect.Parameters["WorldViewProjection"].SetValue(GetWvp());

            DrawGridWithShadow();


            BasicStuff.Draw();
            base.Draw(gameTime);
        }

        // yes its wasteful i just want a working test for now.
        void DrawGridToDepth()
        {
            SetWorld(worldGrid);
            effect.Parameters["lightsPovWorldViewProjection"].SetValue(GetWvp());
            grid.Draw(GraphicsDevice, effect, 0);
            grid.Draw(GraphicsDevice, effect, 1);
            grid.Draw(GraphicsDevice, effect, 2);
        }
        void DrawGridWithShadow()
        {
            SetWorld(worldGrid);
            effect.Parameters["lightsPovWorldViewProjection"].SetValue(GetWvp());
            effect.Parameters["TextureB"].SetValue(renderTargetCube);

            effect.Parameters["TextureA"].SetValue(BasicTextures.red);
            grid.Draw(GraphicsDevice, effect, 0);
            effect.Parameters["TextureA"].SetValue(BasicTextures.green);
            grid.Draw(GraphicsDevice, effect, 1);
            effect.Parameters["TextureA"].SetValue(BasicTextures.blue);
            grid.Draw(GraphicsDevice, effect, 2);
        }

        void SetWorld(Matrix world)
        {
            this.world = world;
        }
        void SetView(Matrix worldForCamera)
        {
            var m = worldForCamera;
            view = Matrix.CreateLookAt(m.Translation, m.Forward + m.Translation, m.Up);
        }
        Matrix GetWvp()
        {
            return world * view * projection;
        }
        void SetViewToLight(Vector3 shadowCubePosition, Vector3 direction)
        {
            Matrix m = Matrix.CreateWorld(shadowCubePosition, direction, Vector3.Up);
            view = Matrix.CreateLookAt(m.Translation, 10f * m.Forward + m.Translation, m.Up);
        }
        Matrix CreateWorldToTarget(Vector3 position, Vector3 targetPosition, Vector3 up)
        {
            return Matrix.CreateWorld(position, targetPosition - position, up);
        }
        

    }
}

Shader.

//
//EffectCubeMap.fx
//
#if OPENGL
    #define SV_POSITION POSITION
    #define VS_SHADERMODEL vs_3_0
    #define PS_SHADERMODEL ps_3_0
#else
    #define VS_SHADERMODEL vs_4_0_level_9_1
    #define PS_SHADERMODEL ps_4_0_level_9_1
#endif

//_______________________
// Common Functions

// Calculate depth
float CalculateLightDepth(float pz, float farPlaneValue)
{
    float result = farPlaneValue - pz;
    return result;
}
// encode 3
float3 EncodeFloatRGB(float f)
{
    float3 color;
    f *= 256;
    color.x = floor(f);
    f = (f - color.x) * 256;
    color.y = floor(f);
    color.z = f - color.y;
    color.xy *= 0.00390625; // *= 1.0/256
    return color;
}
// decode 3
float DecodeFloatRGB(float3 color)
{
    const float3 byte_to_float = float3(1.0, 1.0 / 256, 1.0 / (256 * 256));
    return dot(color, byte_to_float);
}

// _______________________
// texture samplers.

Texture TextureA;
Texture TextureB;
sampler TextureSamplerA = sampler_state
{
    texture = <TextureA>;
    //magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};
sampler TextureSamplerB = sampler_state
{
    texture = <TextureB>;
    //magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};

//_______________________
// specifics

matrix World;
matrix View;
matrix Projection;
matrix WorldViewProjection;
matrix lightsPovWorldViewProjection;

float3  worldLightPosition;
float    farPlane;

//__________________
// Shaders begin now _
// _________________


//
//_______________________________________________________________
// techniques Create depth texture
//_______________________________________________________________
//

struct VsInputCalcSceneDepth
{
    float4 Position : POSITION0;
    float2 TexCoords    : TEXCOORD0;
};
struct VsOutputCalcSceneDepth
{
    float4 Position     : SV_Position;
    float4 Position2D    : TEXCOORD0;
};
struct PsOutputCalcSceneDepth
{
    float4 Color : COLOR0;
};

// ______
// Shader.
//
VsOutputCalcSceneDepth CreateDepthMapVertexShader(VsInputCalcSceneDepth input)//(float4 inPos : POSITION)
{
    VsOutputCalcSceneDepth Output;
    Output.Position = mul(input.Position, lightsPovWorldViewProjection);
    // might need to get changed
    Output.Position2D = Output.Position;
    return Output;
}
//
// I guess we set the render cube side in code so.
// Ok so were going to use the old calculation i guess.
// How does the perspective matrix align with this.
//
PsOutputCalcSceneDepth CreateDepthMapPixelShader(VsOutputCalcSceneDepth input)
{
    PsOutputCalcSceneDepth Output;
    // ABS ?
    float lightDepth = CalculateLightDepth(input.Position2D.z, farPlane);
    float3 temp = EncodeFloatRGB(lightDepth); // 24 bit depth
    float4 result = float4(temp.r, temp.g, temp.b, 1.0f);
    Output.Color = result;
    return Output;
}
//_________
// techniques
technique CreateLightShadowCubeDepth
{
    pass Pass0
    {
        VertexShader = compile VS_SHADERMODEL CreateDepthMapVertexShader();
        PixelShader = compile PS_SHADERMODEL CreateDepthMapPixelShader();
    }
}


//_______________________________________________________________
// techniques DrawLightOrShadow
//_______________________________________________________________
struct VsInLightShadow
{
    float4 Position : POSITION0;
    float3 Normal : NORMAL0;
    float2 TexureCoordinateA : TEXCOORD0;
};
struct VsOutLightShadow
{
    float4 Position : SV_Position;
    float2 TexureCoordinateA : TEXCOORD0;
    float4 Position2D    : TEXCOORD1;
};
struct PsOutLightShadow
{
    float4 Color : COLOR0;
};
//_______
// shaders
//
VsOutLightShadow VsLightShadow(VsInLightShadow input)
{
    VsOutLightShadow output;
    output.Position = mul(input.Position, WorldViewProjection); // basically matrix identity
    output.TexureCoordinateA = input.TexureCoordinateA;
    output.Position2D = output.Position;
    return output;
}
PsOutLightShadow PsLightShadow(VsOutLightShadow input)
{
    PsOutLightShadow output;
    output.Color = tex2D(TextureSamplerA, input.TexureCoordinateA); // *input.Color;
    // ok i think i need to mult the worldLightPosition its really in world space atm.
    float3 lightpos = mul(worldLightPosition, WorldViewProjection);
    float3 dir = input.Position2D - lightpos; //worldLightPosition
    // promlem is i dunno what texcobe does under the hood.
    float shadowDepth = DecodeFloatRGB(texCUBE(TextureSamplerB, dir).xyz);
    // texCUBE(TextureSamplerB, float4(dir, 0)).x;
    if (shadowDepth > input.Position2D.z)
        output.Color = float4(0.7f, 0.7f, 0.7f, 1.0f); // light gray
    return output;
}
//_________
// techniques
technique LightShadowCubeBasicShader
{
    pass
    {
        VertexShader = compile VS_SHADERMODEL VsLightShadow();
        PixelShader = compile PS_SHADERMODEL PsLightShadow();
    }
}

You are using Verctor3.Up for the up vector of the camera. That’s fine for one side of the cube, but not for the others. The up vector needs to be a parameter. For the cube face with direction (0,1,0) for example the up vector needs to be horizontal, so one of those four options: (+/-1,0,0) or (0,0,+/-1). I don’t know which one, that depends on how the faces are laid out on the cubemap. Best to just add the cube faces one by one, and go through the options until it’s right.

There’s also unnecessary math here, this will do the same

void SetViewToLight(Vector3 shadowCubePosition, Vector3 direction, Vector3 up)
{
      view = Matrix.CreateLookAt(shadowCubePosition, shadowCubePosition + direction, up);
}

and then in CreateDepthMapPixelShader

When you transform a vector using a projection matrix, you have to divide result.xyz by result.w, before you have clip space coordinates. So you have to use input.Position2D.z / input.Position2D.w.

If you only need depth in your cubemap, you can just use a single-channel texture format, like SurfaceFormat.Single, and get rid of the RGB conversion. The only reason for converting to RGB is when a single channel doesn’t have enough bits. With floating point textures that’s not necessary anymore.

Nope, you can’t use the screen space position of the pixel, it needs to be in world space, just like lightpos. Just pass worldPos = mul(input.Position, World) down from the vertex shader.

You are comparing shadow depth to pixel depth. It doesn’t work like this, because both depth values are from a different camera. You want to compare shadowDepth to pixel depth, but pixel depth needs to be relative to the light.

I think it’s best to just put world-space depth instead of clip-space depth into the cubemap. Just use pixelDepthFromLight = distance(pixelWorldPos, lightpos), and do the same in CreateDepthMapPixelShader instead of CalculateLightDepth(). By pixelWorldPos I just mean mul(input.Position, World) again.

That’s all I’m seeing for now. To make debugging easier it’s probably a good idea to just display the cubemap on a cube first, before doing shadows, to see if the cubemap is right. If your cube mesh is centered, you can just use the untransformed vertex positions for the texture lookup. Just pass input.position down to the pixel shader, and use it for the texCUBE texture lookup.

Thank’s for taking the time to look this over markus.

Im going to take another shot at this tonight or tommorow when i have time and straighten this up.

I totally forgot about the up orientation that would of caused me a lot of confusion.

I actually switched it over to single, i was trying to copy paste shortcut the space alignment at first and i didn’t fix those methods either so its a mess, i was just to tired to redo it.

How do i return a single from a pixel shader ?

renderTargetCube = new RenderTargetCube(GraphicsDevice, 256, false, SurfaceFormat.Single, DepthFormat.Depth24);
//....

float CreateDepthMapPixelShader(VsOutputCalcSceneDepth input)
{
    //PsOutputCalcSceneDepth output;
    // ABS ?
    float lightDepth = input.Position2D.z / input.Position2D.w ; 
    //float4 result = float4(lightDepth , temp.g, temp.b, 1.0f);
    //output.Color = result;
    //return output;
    /*float4 col = float4(0.0f, 0.0f, 0.0f, 0.0f);
    col.x = lightDepth;
    return col;*/
    return lightDepth;
}

everything i try to do to return it gives a error.

Just make it float4 like normal, the yzw-components will be ignored. You can keep the return lightDepth at the end, as that will be converted to float4(lightDepth,lightDepth ,lightDepth,lightDepth) automatically.

On opengl this will not work.

float4 CreateDepthMapPixelShader(VsOutputCalcSceneDepth input)
{
    float4 lightDepth = input.Position2D / input.Position2D.w; 
    return lightDepth;
}

The only way it seems to work is if i only use a color struct and set the x roundabout
that is super annoying i don’t see why it wouldn’t work…

struct PsOutputCalcSceneDepth
{
    float4 Color : COLOR0;
};

PsOutputCalcSceneDepth CreateDepthMapPixelShader(VsOutputCalcSceneDepth input)
{
    PsOutputCalcSceneDepth output;
    float lightDepth = input.Position2D.z / input.Position2D.w;
    float4 result = float4(0.0f, 0.0f, 0.0f, 0.0f);
    result.x = lightDepth;
    output.Color = result;
    return output;
}

Ah well ill pick it up later.

I think you are just missing the color semantic at the end

float4 CreateDepthMapPixelShader(VsOutputCalcSceneDepth input) : COLOR

I just started working on it again now ill give that a go in a minute.
Edit. well it compiles not sure its actually working though i gotta clean up the cpu side of my test im reorganizing it.

Almost got it now i just have to align the spaces, i think im not sure the output im getting is a little troubling.

Alright i seem to have something off, im getting a entirely shadowed scene but the depth visualization clearly shows that it sees these cubes this has me confused.

Im not sure either if the light position is truely at the cubes center or what is going on.

Heres my game1 and heres my shader currently.

game1

using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Input;
using System;

namespace TestTextureCube
{

    public class Game1 : Game
    {
        GraphicsDeviceManager graphics;
        SpriteBatch spriteBatch;
        RenderTargetCube renderTargetCube;
        Effect effect;
        //
        float near = .1f;
        float far = 1000f;
        //
        Matrix world, view, projection;
        Matrix worldGrid, worldCamera, worldLight, worldCube;
        // well draw a bunch.
        Matrix[] worldCubes;
        int numberOfCubes = 30;
        Vector3 theLightsViewPosition;

        // something to draw
        Grid3dOrientation grid = new Grid3dOrientation(30, 30, .001f);
        CubePNTT cube = new CubePNTT(true); 
        Quad quad = new Quad(1.7f); // 2.0 is screen scale with a wvp identity matrix.

        public Game1()
        {
            graphics = new GraphicsDeviceManager(this);
            Content.RootDirectory = "Content";
        }

        protected override void Initialize()
        {
            base.Initialize();
        }

        protected override void LoadContent()
        {
            spriteBatch = this.LoadBasics("Shadow Cube test");
            effect = Content.Load<Effect>("EffectCubeMap");
            renderTargetCube = new RenderTargetCube(GraphicsDevice, 256, false, SurfaceFormat.Single, DepthFormat.Depth24);
            //effect.Parameters["farPlane"].SetValue(far);

            world = Matrix.Identity;
            SetCreateView(world);
            SetProjection(Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * .5f, GraphicsDevice.Viewport.Width / GraphicsDevice.Viewport.Height, near, far));

            worldLight = CreateWorldToTarget(new Vector3(3f, 2f, -2f), Vector3.Zero, Vector3.Up);
            worldGrid = Matrix.CreateScale(30f) * Matrix.CreateWorld(Vector3.Zero, Vector3.Forward, Vector3.Up);
            worldCube = Matrix.CreateScale(1f) * CreateWorldToTarget(new Vector3(3f, .1f, -.5f), Vector3.Zero, Vector3.Up);
            worldCamera = CreateWorldToTarget(new Vector3(.5f, -.5f, 4f), Vector3.Zero, Vector3.Up);

            MakeBunchesOfCubes();

            BasicStuff.camera.CameraType(BasicCamera.MOVEMENT_CAMERA);
            BasicStuff.camera.SetPosition(worldCamera.Translation);
        }

        void MakeBunchesOfCubes()
        {
            // spiral
            var pi2 = MathHelper.Pi *2f;
            int len = numberOfCubes;
            worldCubes = new Matrix[len];
            for(int i = 0; i < len; i++)
            {
                float r = (float)(i) / (float)(len);
                var t = (r) * pi2 * 2f; // 12 radians 2 circles
                var x = (float)Math.Sin(t);
                var y = 0f;
                var z = (float)Math.Cos(t);
                Vector3 p = new Vector3(x,y,z) * (2f + r * 4f ) + worldLight.Translation; // spirals instead
                worldCubes[i] = Matrix.CreateScale(.4f + r) * Matrix.CreateWorld(p, Vector3.Forward, Vector3.Up);
            }
        }

        protected override void UnloadContent()
        {
            this.UnloadBasics();
        }

        protected override void Update(GameTime gameTime)
        {
            if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed || Keyboard.GetState().IsKeyDown(Keys.Escape))
                Exit();

            BasicStuff.Update(gameTime);
            base.Update(gameTime);
        }

        protected override void Draw(GameTime gameTime)
        {
            GraphicsDevice.Clear(Color.Black);
            GraphicsDevice.Clear(ClearOptions.DepthBuffer, Color.Black, 1, 0);
            GraphicsDevice.RasterizerState = new RasterizerState() { FillMode = FillMode.Solid, CullMode = CullMode.None };
            GraphicsDevice.DepthStencilState = new DepthStencilState() { DepthBufferEnable = true, DepthBufferFunction = CompareFunction.LessEqual };

            //
            // Render to each face of the cube this will need a spacial partition of some type later on.
            //
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeX);
            SetCreateView(worldLight.Translation, Vector3.Left, Vector3.Up);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeY);
            SetCreateView(worldLight.Translation, Vector3.Down, Vector3.Forward);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeZ);
            SetCreateView(worldLight.Translation, Vector3.Forward, Vector3.Up);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveX);
            SetCreateView(worldLight.Translation, Vector3.Right, Vector3.Up);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveY);
            SetCreateView(worldLight.Translation, Vector3.Up, Vector3.Backward);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveZ);
            SetCreateView(worldLight.Translation, Vector3.Backward, Vector3.Up);
            RenderSceneDepth();

            //
            // Draw the scene with shadow from our depth cube.       
            //
            GraphicsDevice.SetRenderTarget(null);  // Switch back to the buffer.
            SetCreateView(BasicStuff.camera.World); // We switch our view to the scene camera.

            // Draw the scene with shadow.
            DrawSceneWithShadowing(); 

            // Draw the visualization enviromental cube. ,  note don't bother rotating it into place
            DrawVisualizationCube();

            base.Draw(gameTime);
        }

        public void RenderSceneDepth()
        {
            effect.CurrentTechnique = effect.Techniques["RenderShadowDepth"];
            effect.Parameters["worldLightPosition"].SetValue(worldLight.Translation);

            //// render grids depth  // Dont render this depth for the moment.
            //SetWorld(worldGrid);
            //grid.Draw(GraphicsDevice, effect, 0);
            //grid.Draw(GraphicsDevice, effect, 1);
            //grid.Draw(GraphicsDevice, effect, 2);

            // render cubes depth
            effect.Parameters["TextureA"].SetValue(BasicTextures.checkerBoard);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            for (int i = 0; i < worldCubes.Length;i++)
            {
                SetWorld(worldCubes[i]);
                cube.Draw(GraphicsDevice, effect);
            }
        }
        public void DrawSceneWithShadowing()
        {
            effect.CurrentTechnique = effect.Techniques["LightShadowCubeBasicShader"];
            effect.Parameters["worldLightPosition"].SetValue(worldLight.Translation);
            SetCreateView(BasicStuff.camera.World); // We switch our view to camera.
            // draw grid
            SetWorld(worldGrid);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            //
            effect.Parameters["TextureA"].SetValue(BasicTextures.red);
            grid.Draw(GraphicsDevice, effect, 0);
            effect.Parameters["TextureA"].SetValue(BasicTextures.green);
            grid.Draw(GraphicsDevice, effect, 1);
            effect.Parameters["TextureA"].SetValue(BasicTextures.blue);
            grid.Draw(GraphicsDevice, effect, 2);

            // draw cubes
            effect.Parameters["TextureA"].SetValue(BasicTextures.checkerBoard);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            for (int i = 0; i < worldCubes.Length; i++)
            {
                SetWorld(worldCubes[i]);
                cube.Draw(GraphicsDevice, effect);
            }
        }

        public void DrawVisualizationCube()
        {
            // draw a cube at the light with the depth visualization
            effect.CurrentTechnique = effect.Techniques["DepthVisualization"];
            SetWorld(worldLight);
            effect.Parameters["TextureA"].SetValue(BasicTextures.checkerBoard);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            cube.Draw(GraphicsDevice, effect);
        }

        //____________________
        // some extra helper stuff
        // ___________________

        Matrix CreateWorldToTarget(Vector3 position, Vector3 targetPosition, Vector3 up)
        {
            return Matrix.CreateWorld(position, targetPosition - position, up);
        }
        void SetWorld(Matrix World)
        {
            world = World;
            effect.Parameters["World"].SetValue(world);
        }
        void SetCreateView(Matrix worldForCamera)
        {
            view = Matrix.CreateLookAt(worldForCamera.Translation, worldForCamera.Forward + worldForCamera.Translation, worldForCamera.Up);
            effect.Parameters["View"].SetValue(view);
        }
        void SetCreateView(Vector3 position, Vector3 direction, Vector3 up)
        {
            view = Matrix.CreateLookAt(position, position + direction, up);
            effect.Parameters["View"].SetValue(view);
        }
        Vector3 GetViewPosition()
        {
            return view.Translation;
        }
        // humm should i be using the lights view position because of the targetcube.
        Vector3 GetLightPosition(Matrix theLightsWorldMatrix)
        {
            var temp = Matrix.CreateLookAt(theLightsWorldMatrix.Translation, theLightsWorldMatrix.Forward + theLightsWorldMatrix.Translation, theLightsWorldMatrix.Up);
            return temp.Translation;
        }
        void SetProjection(Matrix Projection)
        {
            projection = Projection;
            effect.Parameters["Projection"].SetValue(projection);
        }
        void SetCreatePerspectiveProjection(float near, float far, float fovInDegrees)
        {
            projection= Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * MathHelper.ToRadians(fovInDegrees), GraphicsDevice.Viewport.Width / GraphicsDevice.Viewport.Height, near, far);
            effect.Parameters["Projection"].SetValue(projection);
        }
    }
}

shader

//____________________
//EffectCubeMap.fx
//____________________
#if OPENGL
    #define SV_POSITION POSITION
    #define VS_SHADERMODEL vs_3_0
    #define PS_SHADERMODEL ps_3_0
#else
    #define VS_SHADERMODEL vs_4_0_level_9_1
    #define PS_SHADERMODEL ps_4_0_level_9_1
#endif

//_______________________
// Common Functions
//_______________________

// _______________________
// Texture samplers.
//_______________________
Texture TextureA; // primary texture.
Texture TextureB; // render texture or secondary multi texture.
sampler TextureSamplerA = sampler_state
{
    texture = <TextureA>;
    //magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};
sampler TextureSamplerB = sampler_state
{
    texture = <TextureB>;
    //magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};

//_______________________
// Shader globals
//_______________________
matrix World;
matrix View;
matrix Projection;
float3  worldLightPosition;

//_______________________________________________________________
// technique
// Render shadow depth
//_______________________________________________________________/
struct VsInputCalcSceneDepth
{
    float4 Position : POSITION0;
    float2 TexCoords    : TEXCOORD0;
};
struct VsOutputCalcSceneDepth
{
    float4 Position     : SV_Position;
    float4 Position3D    : TEXCOORD0;
};
// Shader.
VsOutputCalcSceneDepth CreateDepthMapVertexShader(VsInputCalcSceneDepth input)//(float4 inPos : POSITION)
{
    VsOutputCalcSceneDepth output;
    output.Position3D = mul(input.Position, World);
    float4x4 wvp = mul(World, mul(View, Projection));
    output.Position = mul(input.Position, wvp);
    return output;
}
float4 CreateDepthMapPixelShader(VsOutputCalcSceneDepth input) : COLOR
{
    float lightDepth = input.Position3D.z; 
    return lightDepth;
}
//_________
// techniques
technique RenderShadowDepth
{
    pass Pass0
    {
        VertexShader = compile VS_SHADERMODEL CreateDepthMapVertexShader();
        PixelShader = compile PS_SHADERMODEL CreateDepthMapPixelShader();
    }
}


//_______________________________________________________________
// technique
// DrawLightOrShadow
//_______________________________________________________________
struct VsInLightShadow
{
    float4 Position : POSITION0;
    float3 Normal : NORMAL0;
    float2 TexureCoordinateA : TEXCOORD0;
};
struct VsOutLightShadow
{
    float4 Position : SV_Position;
    float2 TexureCoordinateA : TEXCOORD0;
    float4 Position3D    : TEXCOORD1;
};
struct PsOutLightShadow
{
    float4 Color : COLOR0;
};
// shaders
VsOutLightShadow VsLightShadow(VsInLightShadow input)
{
    VsOutLightShadow output;
    output.Position3D = mul(input.Position, World);
    float4x4 wvp = mul(World, mul(View, Projection));
    output.Position = mul(input.Position, wvp);
    output.TexureCoordinateA = input.TexureCoordinateA;
    return output;
}
PsOutLightShadow PsLightShadow(VsOutLightShadow input)
{
    PsOutLightShadow output;
    float3 pixelpos = input.Position3D;
    float3 lightpos = worldLightPosition;
    float3 lightToPixelDir = pixelpos - lightpos;
    float pixelDistance = distance(pixelpos, lightpos);
    float4 texelcolor = tex2D(TextureSamplerA, input.TexureCoordinateA); // *input.Color;
    float shadowDepth = texCUBE(TextureSamplerB, float4(lightToPixelDir, 0)).x;
    if (shadowDepth < pixelDistance)
        texelcolor = float4(0.7f, 0.7f, 0.7f, 1.0f); // light gray
    output.Color = texelcolor;
    return output;
}
//_________
// techniques
technique LightShadowCubeBasicShader
{
    pass
    {
        VertexShader = compile VS_SHADERMODEL VsLightShadow();
        PixelShader = compile PS_SHADERMODEL PsLightShadow();
    }
}


//_______________________________________________________________
// technique 
// Depth Visualization
//_______________________________________________________________
// shaders
VsOutLightShadow VsDepthVisualization(VsInLightShadow input)
{
    VsOutLightShadow output;
    output.Position3D = mul(input.Position, World);
    float4x4 wvp = mul(World, mul(View, Projection));
    output.Position = mul(input.Position, wvp);
    output.TexureCoordinateA = input.TexureCoordinateA;
    return output;
}
PsOutLightShadow PsDepthVisualization(VsOutLightShadow input)
{
    PsOutLightShadow output;
    float3 pixelpos = input.Position3D.xyz; // / input.Position3D.w;
    float3 lightpos = worldLightPosition;
    float3 lightToPixelDir = pixelpos - lightpos;
    float4 texelcolor = tex2D(TextureSamplerA, input.TexureCoordinateA);
    float shadowDepth = texCUBE(TextureSamplerB, float4(lightToPixelDir, 0)).x;
    texelcolor = (texelcolor + shadowDepth) * 0.5f;
    output.Color = texelcolor; 
    return output;
}
technique DepthVisualization
{
    pass
    {
        VertexShader = compile VS_SHADERMODEL VsDepthVisualization();
        PixelShader = compile PS_SHADERMODEL PsDepthVisualization();
    }
}

Now if i change this.

pixelposition despite the name and all of the comparison variables should be in world space.

/*if (shadowDepth < pixelDistance)*/
if (shadowDepth < pixelpos.z)

i get this below…

What is even happening here i have no idea.
As the grid lines are not being depth rendered.

I can zip up the project, if you want to take a look at the whole thing.
Newer bug test is below.

OK, I got it working. Here are the main things I had to change:

  • You used the normal scene camera projection matrix also for the shadowmap. The shadowmap camera needs to have aspect ratio 1.

  • Return distance(worldLightPosition, input.Position3D) instead of Position3D.z in CreateDepthMapPixelShader.

  • Add a bit of an offset to avoid z-fighting In PsLightShadow:

if (shadowDepth*1.01 < lightToPixelDist)
  • Removed all rotations from the DepthVisualization shader. The shadowmap cube needs to be world-axis aligned.

  • The RenderTargetCube needs to be cleared to max depth rather than black. Since you can’t clear to values bigger than 1 with the default clearscreen (you need a custom shader), I just surrounded the scene with a huge cube in RenderSceneDepth.

The top and bottom faces might still need the up vector corrected. There’s nothing above or below the light, so I couldn’t check that.

The corrected game class:

using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Input;
using System;

namespace TestTextureCube
{

    public class Game1 : Game
    {
        GraphicsDeviceManager graphics;
        SpriteBatch spriteBatch;
        RenderTargetCube renderTargetCube;
        Effect effect;
        //
        float near = .1f;
        float far = 1000f;
        //
        Matrix world, view, projection;
        Matrix worldGrid, worldCamera, worldLight, worldCube;

        Matrix mainProjection, shadowMapProjection;

        // well draw a bunch.
        Matrix[] worldCubes;
        int numberOfCubes = 30;
        Vector3 theLightsViewPosition;

        // something to draw
        Grid3dOrientation grid = new Grid3dOrientation(30, 30, .001f);
        CubePNTT cube = new CubePNTT(true); 
        Quad quad = new Quad(1.7f); // 2.0 is screen scale with a wvp identity matrix.

        public Game1()
        {
            graphics = new GraphicsDeviceManager(this);
            Content.RootDirectory = "Content";
        }

        protected override void Initialize()
        {
            base.Initialize();
        }

        protected override void LoadContent()
        {
            spriteBatch = this.LoadBasics("Shadow Cube test");
            effect = Content.Load<Effect>("EffectCubeMap");
            renderTargetCube = new RenderTargetCube(GraphicsDevice, 512, false, SurfaceFormat.Single, DepthFormat.Depth24);
            //effect.Parameters["farPlane"].SetValue(far);

            world = Matrix.Identity;
            SetCreateView(world);

            mainProjection = Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * .5f, (float)GraphicsDevice.Viewport.Width / GraphicsDevice.Viewport.Height, near, far);
            shadowMapProjection = Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * .5f, 1, near, far);

            worldLight = CreateWorldToTarget(new Vector3(3f, 2f, -2f), Vector3.Zero, Vector3.Up);
            worldGrid = Matrix.CreateScale(30f) * Matrix.CreateWorld(Vector3.Zero, Vector3.Forward, Vector3.Up);
            worldCube = Matrix.CreateScale(1f) * CreateWorldToTarget(new Vector3(3f, .1f, -.5f), Vector3.Zero, Vector3.Up);
            worldCamera = CreateWorldToTarget(new Vector3(.5f, -.5f, 4f), Vector3.Zero, Vector3.Up);

            MakeBunchesOfCubes();

            BasicStuff.camera.CameraType(BasicCamera.MOVEMENT_CAMERA);
            BasicStuff.camera.SetPosition(worldCamera.Translation);
        }

        void MakeBunchesOfCubes()
        {
            // spiral
            var pi2 = MathHelper.Pi *2f;
            int len = numberOfCubes;
            worldCubes = new Matrix[len];
            for(int i = 0; i < len; i++)
            {
                float r = (float)(i) / (float)(len);
                var t = (r) * pi2 * 2f; // 12 radians 2 circles
                var x = (float)Math.Sin(t);
                var y = 0f;
                var z = (float)Math.Cos(t);
                Vector3 p = new Vector3(x,y,z) * (1f + r * 5f ) + worldLight.Translation; // spirals instead
                worldCubes[i] = Matrix.CreateScale(.05f + r * 1.6f) * Matrix.CreateWorld(p, Vector3.Forward, Vector3.Up);
            }
        }

        protected override void UnloadContent()
        {
            this.UnloadBasics();
        }

        protected override void Update(GameTime gameTime)
        {
            if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed || Keyboard.GetState().IsKeyDown(Keys.Escape))
                Exit();

            BasicStuff.Update(gameTime);
            base.Update(gameTime);
        }

        protected override void Draw(GameTime gameTime)
        {
            SetProjection(shadowMapProjection);

            GraphicsDevice.Clear(Color.Black);
            GraphicsDevice.Clear(ClearOptions.DepthBuffer, Color.Black, 1, 0);
            GraphicsDevice.RasterizerState = new RasterizerState() { FillMode = FillMode.Solid, CullMode = CullMode.None };
            GraphicsDevice.DepthStencilState = new DepthStencilState() { DepthBufferEnable = true, DepthBufferFunction = CompareFunction.LessEqual };

            //
            // Render to each face of the cube this will need a spacial partition of some type later on. 
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeX);
            SetCreateView(worldLight.Translation, Vector3.Left, Vector3.Down);
            RenderSceneDepth();    
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeY);
            SetCreateView(worldLight.Translation, Vector3.Down, Vector3.Forward);
            RenderSceneDepth();     
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeZ);
            SetCreateView(worldLight.Translation, Vector3.Forward, Vector3.Down);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveX);
            SetCreateView(worldLight.Translation, Vector3.Right, Vector3.Down);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveY);
            SetCreateView(worldLight.Translation, Vector3.Up, Vector3.Backward);
            RenderSceneDepth();
            GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveZ);
            SetCreateView(worldLight.Translation, Vector3.Backward, Vector3.Down);
            RenderSceneDepth();

            //
            // Draw the scene with shadow from our depth cube.       
            //
            SetProjection(mainProjection);
            GraphicsDevice.SetRenderTarget(null);  // Switch back to the buffer.
            SetCreateView(BasicStuff.camera.World); // We switch our view to the scene camera.

            // Draw the scene with shadow.
            DrawSceneWithShadowing(); 

            // Draw the visualization enviromental cube. ,  note don't bother rotating it into place
            DrawVisualizationCube();

            base.Draw(gameTime);
        }

        public void RenderSceneDepth()
        {
            effect.CurrentTechnique = effect.Techniques["RenderShadowDepth"];
            effect.Parameters["worldLightPosition"].SetValue(worldLight.Translation);

            //// render grids depth  // we wont render this depth for the moment.
            //SetWorld(worldGrid);
            //grid.Draw(GraphicsDevice, effect, 0);
            //grid.Draw(GraphicsDevice, effect, 1);
            //grid.Draw(GraphicsDevice, effect, 2);

            // render cubes depth
            //effect.Parameters["TextureA"].SetValue(BasicTextures.checkerBoard);
            //effect.Parameters["TextureB"].SetValue(renderTargetCube);

            // draw huge background cube, replaces clearing the rendertarget to maxdepth
            SetWorld(Matrix.CreateScale(far/2));
            cube.Draw(GraphicsDevice, effect);

            // draw regular cubes
            for (int i = 0; i < worldCubes.Length;i++)
            {
                SetWorld(worldCubes[i]);
                cube.Draw(GraphicsDevice, effect);
            }
        }
        public void DrawSceneWithShadowing()
        {
            effect.CurrentTechnique = effect.Techniques["LightShadowCubeBasicShader"];
            effect.Parameters["worldLightPosition"].SetValue(worldLight.Translation);
            SetCreateView(BasicStuff.camera.World); // We switch our view to camera.
            // draw grid
            SetWorld(worldGrid);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            //
            effect.Parameters["TextureA"].SetValue(BasicTextures.red);
            grid.Draw(GraphicsDevice, effect, 0);
            effect.Parameters["TextureA"].SetValue(BasicTextures.green);
            grid.Draw(GraphicsDevice, effect, 1);
            effect.Parameters["TextureA"].SetValue(BasicTextures.blue);
            grid.Draw(GraphicsDevice, effect, 2);

            // draw cubes
            effect.Parameters["TextureA"].SetValue(BasicTextures.checkerBoard);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            for (int i = 0; i < worldCubes.Length; i++)
            {
                SetWorld(worldCubes[i]);
                cube.Draw(GraphicsDevice, effect);
            }
        }

        public void DrawVisualizationCube()
        {
            // draw a cube at the light with the depth visualization
            effect.CurrentTechnique = effect.Techniques["DepthVisualization"];
            effect.Parameters["worldLightPosition"].SetValue(worldLight.Translation);
            effect.Parameters["TextureB"].SetValue(renderTargetCube);
            cube.Draw(GraphicsDevice, effect);
        }

        //____________________
        // some extra helper stuff
        // ___________________

        Matrix CreateWorldToTarget(Vector3 position, Vector3 targetPosition, Vector3 up)
        {
            return Matrix.CreateWorld(position, targetPosition - position, up);
        }
        void SetWorld(Matrix World)
        {
            world = World;
            effect.Parameters["World"].SetValue(world);
        }
        void SetCreateView(Matrix worldForCamera)
        {
            view = Matrix.CreateLookAt(worldForCamera.Translation, worldForCamera.Forward + worldForCamera.Translation, worldForCamera.Up);
            effect.Parameters["View"].SetValue(view);
        }
        void SetCreateView(Vector3 position, Vector3 direction, Vector3 up)
        {
            view = Matrix.CreateLookAt(position, position + direction, up);
            effect.Parameters["View"].SetValue(view);
        }
        Vector3 GetViewPosition()
        {
            return view.Translation;
        }
        // humm im not to sure about this.
        Vector3 GetLightPosition(Matrix theLightsWorldMatrix)
        {
            var temp = Matrix.CreateLookAt(theLightsWorldMatrix.Translation, theLightsWorldMatrix.Forward + theLightsWorldMatrix.Translation, theLightsWorldMatrix.Up);
            return temp.Translation;
        }
        void SetProjection(Matrix Projection)
        {
            projection = Projection;
            effect.Parameters["Projection"].SetValue(projection);
        }
        void SetCreatePerspectiveProjection(float near, float far, float fovInDegrees)
        {
            projection= Matrix.CreatePerspectiveFieldOfView((float)MathHelper.Pi * MathHelper.ToRadians(fovInDegrees), GraphicsDevice.Viewport.Width / GraphicsDevice.Viewport.Height, near, far);
            effect.Parameters["Projection"].SetValue(projection);
        }
        


    }
}

and the corrected shader:

//____________________
//EffectCubeMap.fx
//____________________
#if OPENGL
	#define SV_POSITION POSITION
	#define VS_SHADERMODEL vs_3_0
	#define PS_SHADERMODEL ps_3_0
#else
	#define VS_SHADERMODEL vs_4_0_level_9_1
	#define PS_SHADERMODEL ps_4_0_level_9_1
#endif

//_______________________
// Common Functions
//_______________________

// _______________________
// Texture samplers.
//_______________________
Texture TextureA; // primary texture.
Texture TextureB; // render texture or secondary multi texture.
sampler TextureSamplerA = sampler_state
{
	texture = <TextureA>;
	//magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};
sampler TextureSamplerB = sampler_state
{
	texture = <TextureB>;
	//magfilter = LINEAR; //minfilter = LINEAR; //mipfilter = LINEAR; //AddressU = mirror; //AddressV = mirror; 
};

//_______________________
// Shader globals
//_______________________
matrix World;
matrix View;
matrix Projection;
float3  worldLightPosition;

//_______________________________________________________________
// technique
// Render shadow depth
//_______________________________________________________________/
struct VsInputCalcSceneDepth
{
	float4 Position : POSITION0;
};
struct VsOutputCalcSceneDepth
{
	float4 Position     : SV_Position;
	float4 Position3D    : TEXCOORD0;
};
// Shader.
VsOutputCalcSceneDepth CreateDepthMapVertexShader(VsInputCalcSceneDepth input)//(float4 inPos : POSITION)
{
	VsOutputCalcSceneDepth output;
	output.Position3D = mul(input.Position, World);
	float4x4 vp = mul(View, Projection);
	output.Position = mul(output.Position3D, vp);
	
	return output;
}
float4 CreateDepthMapPixelShader(VsOutputCalcSceneDepth input) : COLOR
{
	return distance(worldLightPosition, input.Position3D);
}
//_________
// techniques
technique RenderShadowDepth
{
	pass Pass0
	{
		VertexShader = compile VS_SHADERMODEL CreateDepthMapVertexShader();
		PixelShader = compile PS_SHADERMODEL CreateDepthMapPixelShader();
	}
}


//_______________________________________________________________
// technique
// DrawLightOrShadow
//_______________________________________________________________
struct VsInLightShadow
{
	float4 Position : POSITION0;
	float3 Normal : NORMAL0;
	float2 TexureCoordinateA : TEXCOORD0;
};
struct VsOutLightShadow
{
	float4 Position : SV_Position;
	float2 TexureCoordinateA : TEXCOORD0;
	float4 Position3D    : TEXCOORD1;
};
struct PsOutLightShadow
{
	float4 Color : COLOR0;
};
// shaders
VsOutLightShadow VsLightShadow(VsInLightShadow input)
{
	VsOutLightShadow output;
	output.Position3D = mul(input.Position, World);
	float4x4 vp = mul(View, Projection);
	output.Position = mul(output.Position3D, vp);
	output.TexureCoordinateA = input.TexureCoordinateA;
	return output;
}
PsOutLightShadow PsLightShadow(VsOutLightShadow input)
{
	PsOutLightShadow output;
	float3 pixelpos = input.Position3D.xyz;
	float3 lightpos = worldLightPosition;
	float3 lightToPixelDir = pixelpos - lightpos;
	float lightToPixelDist = length(lightToPixelDir);
	float4 texelcolor = tex2D(TextureSamplerA, input.TexureCoordinateA); // *input.Color;
	float shadowDepth = texCUBE(TextureSamplerB, float4(lightToPixelDir, 0)).x;
	if (shadowDepth*1.01 < lightToPixelDist)
		texelcolor *= float4(0.3f, 0.3f, 0.3f, 1.0f); // dark gray
	output.Color = texelcolor;
	return output;
}
//_________
// techniques
technique LightShadowCubeBasicShader
{
	pass
	{
		VertexShader = compile VS_SHADERMODEL VsLightShadow();
		PixelShader = compile PS_SHADERMODEL PsLightShadow();
	}
}


//_______________________________________________________________
// technique 
// Depth Visualization
//_______________________________________________________________
// shaders
struct VsOutDepthVisualization
{
	float4 Position : SV_Position;
	float3 dir : TEXCOORD0;
};

VsOutDepthVisualization VsDepthVisualization(VsInLightShadow input)
{
	VsOutDepthVisualization output;
	float4 worldPos = float4(worldLightPosition + input.Position, 1);
	float4x4 vp = mul(View, Projection);
	output.Position = mul(worldPos, vp);
	output.dir = input.Position;
	return output;
}
float4 PsDepthVisualization(VsOutDepthVisualization input) : COLOR
{
	float shadowDepth = texCUBE(TextureSamplerB, float4(input.dir, 0)).x;
	return shadowDepth * 0.1;
}

technique DepthVisualization
{
	pass
	{
		VertexShader = compile VS_SHADERMODEL VsDepthVisualization();
		PixelShader = compile PS_SHADERMODEL PsDepthVisualization();
	}
}

//_______________________________________________________________
// techniques 
// Quad Draw
//_______________________________________________________________
struct VsInputQuad
{
	float4 Position : POSITION0;
	float3 Normal : NORMAL0;
	float2 TexureCoordinateA : TEXCOORD0;
};
struct VsOutputQuad
{
	float4 Position : SV_Position;
	float2 TexureCoordinateA : TEXCOORD0;
};
struct PsOutputQuad
{
	float4 Color : COLOR0;
};
// ____________________________
VsOutputQuad VertexShaderQuadDraw(VsInputQuad input)
{
	VsOutputQuad output;
	float4x4 wvp = mul(World, mul(View, Projection));
	output.Position = mul(input.Position, wvp); // basically matrix identity
	output.TexureCoordinateA = input.TexureCoordinateA;
	return output;
}
PsOutputQuad PixelShaderQuadDraw(VsOutputQuad input)
{
	PsOutputQuad output;
	output.Color = tex2D(TextureSamplerA, input.TexureCoordinateA); // *input.Color;
	return output;
}
technique QuadDraw
{
	pass
	{
		VertexShader = compile VS_SHADERMODEL VertexShaderQuadDraw();
		PixelShader = compile PS_SHADERMODEL PixelShaderQuadDraw();
	}
}

//__________________________

//struct VertexShaderInput
//{
//	float4 Position : POSITION0;
//	float4 Color : COLOR0;
//};
//
//struct VertexShaderOutput
//{
//	float4 Position : SV_POSITION;
//	float4 Color : COLOR0;
//};
//
//VertexShaderOutput MainVS(in VertexShaderInput input)
//{
//	VertexShaderOutput output = (VertexShaderOutput)0;
//
//	output.Position = mul(input.Position, WorldViewProjection);
//	output.Color = input.Color;
//
//	return output;
//}
//
//float4 MainPS(VertexShaderOutput input) : COLOR
//{
//	return input.Color;
//}
//
//technique BasicColorDrawing
//{
//	pass P0
//	{
//		VertexShader = compile VS_SHADERMODEL MainVS();
//		PixelShader = compile PS_SHADERMODEL MainPS();
//	}
//};

EDIT
Fixed the aspect ratio for the main camera as well. Wit the integer division you are loosing everything after the comma.

Edit :confused:

Oh snap it almost works.

I was racking my brain till my eyeballs were poping out.

Ill have to wait till i have more time later, to study the changes you made.
Ill probably simplify it later on and leave a trimmed down example for others as a reference.

Thanks so much markus.

Ah ok this is not so solved as i thought.
Last night i had some time to mess with this again and i was double checking it after i drew the grid lines and they were off.

So i altered the cube draw to be off tilt to check it.
i got the below.

No problem i thought ill manually fit each cube face to place.
I figured the view would have to be tweeked anyways.

but …

What i didn’t expect is that the coordinates couldn’t be tweeked no matter what orthagonal up i placed for the forward view that corresponds to the negativeZ render cube, it appears to be giving me back inverse Y coordinates or inverse X coordinates no matter what up i set for the view matrix.

No matter how i change it, if i use the up down right or left on every one of them i get a bad mapping to the shadow u,v depth texture.

Not sure how to handle this if i should try to transpose the matrix’s before sending them, try that on the shader, or reverse the direction and flip the view forward to the positive z render ? I can’t manually invert the u or v on the render cube.

Or is it possible im missing something simple entirely?

.
.
.

ill upload the newer test in a minute if anyone has any thoughts so they can run it themselves.
It’s messier but the first ones in there too, i fixed a couple things in between like the cube so its properly aligned to a regular cube or skybox with individual textures makes it easier to see.

Edit: tried everything i could think off the forward alone only appears to align with a up of right but it still wont align with the shadow. Doing it the old way i can’t transpose the matrix to make it align in the shader… Im stumped i looked at kosmotos shader but hes using a orthographic. projection.

I think the problem is that Matrix.CreateLookAt creates a right-handed view matrix, but those cubemaps want left-handed DirectX coordinates. You could fix it in the texCUBE lookup, but you can also just make a left-handed view matrix like that:

void SetCreateCubeMapView(Vector3 position, Vector3 direction, Vector3 up)
{
     Matrix view = Matrix.Identity;
     view.Translation = position;
     view.Forward = direction;
     view.Up = up;
     view.Right = Vector3.Cross(up, direction);
     view = Matrix.Invert(view);
     effect.Parameters["View"].SetValue(view);
}

You should be able to optimize this to get rid of the matrix inversion, but it works. You can then create the individual cube faces like that:

GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeX);
SetCreateCubeMapView(worldLight.Translation, Vector3.Left, Vector3.Up);
RenderSceneDepth();
GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeY); // bottom
SetCreateCubeMapView(worldLight.Translation, Vector3.Down, Vector3.Backward); // bottom 
RenderSceneDepth();
GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.NegativeZ); // front 
SetCreateCubeMapView(worldLight.Translation, Vector3.Forward, Vector3.Up);// front  
RenderSceneDepth();
GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveX);
SetCreateCubeMapView(worldLight.Translation, Vector3.Right, Vector3.Up);
RenderSceneDepth();
GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveY); // top
SetCreateCubeMapView(worldLight.Translation, Vector3.Up, Vector3.Forward); // top 
RenderSceneDepth();
GraphicsDevice.SetRenderTarget(renderTargetCube, CubeMapFace.PositiveZ);
SetCreateCubeMapView(worldLight.Translation, Vector3.Backward, Vector3.Up);
RenderSceneDepth();
1 Like

Thats a world matrix with the right negated then inverted also.
Humm
I can just hard code the values and pre-compute them as long as i don’t want to spin the rendering.
but…

Shouldn’t this be considered a bug ?

That its very much un-intuitive behavior at the least.
This is the same on dx as well as gl.

Well im glad your here cause i don’t think i would of figured it out i was trying all kinds of stuff.

That’s what a view matrix basically is. It’s just the camera’s world matrix, but inverted. It makes a lot of sense when you think about it. If your camera moves towards the right, your view of the world moves towards the left. If you camera rotates clockwise, your view rotates counterclockwise. Your view always does the exact opposite of you camera, that’s what the inversion is for.

It definitely looks like a coordinate system inconsistency. I don’t know what the exact backstory is here.

@kosmonautgames I’d be interested to hear if you experienced the same. Did you also have to make you view matrix left-handed, or, change the lookup-direction in texCUBE to compensate for left/right handedness?

Ya but the view matrix alters the position a bit if i remember right were the create world doesn’'t well not on its own it doesn’t.

Im still a little worried about how that might affect the uv calculation on the cube under perspective.
Now that its clear whats going on though.
I suppose if it really turns out to be a problem i can find the direct dx create LH LookAt function the code is on the msdn somewere i found it before.

Still i feel like this needs documentation or a outright helper method in the rendertargetcube itself or something to make this clear. I mean this is going to end up a extension method for sure for me.

I think it’s fine, you’ve probably just looked at view matrix code that optimized away the inversion.

Or just take the Monogame CreateLookat code and negate the right vector.

Agreed.

Thanks again markus.

I put up a issue about this on github.

just take the Monogame CreateLookat code and negate the right vector.

Not sure this is right but it appears to be.

        public Matrix CreateLookAtLeftHanded( Vector3 cameraPosition,  Vector3 cameraTarget,  Vector3 cameraUpVector)
        {
            var vector = Vector3.Normalize(cameraPosition - cameraTarget);
            var vector2 = -Vector3.Normalize(Vector3.Cross(cameraUpVector, vector));
            var vector3 = Vector3.Cross(-vector, vector2);
            Matrix result = Matrix.Identity;
            result.M11 = vector2.X;
            result.M12 = vector3.X;
            result.M13 = vector.X;
            result.M14 = 0f;
            result.M21 = vector2.Y;
            result.M22 = vector3.Y;
            result.M23 = vector.Y;
            result.M24 = 0f;
            result.M31 = vector2.Z;
            result.M32 = vector3.Z;
            result.M33 = vector.Z;
            result.M34 = 0f;
            result.M41 = -Vector3.Dot(vector2, cameraPosition);
            result.M42 = -Vector3.Dot(vector3, cameraPosition);
            result.M43 = -Vector3.Dot(vector, cameraPosition);
            result.M44 = 1f;
            return result;
        }

Here is a Desktop GL demo version that demonstrates the results of the post for reference.

I also found a inconsistancy on Dx.
Within the shader this struct will run on GL uncommented but not on DX.

struct VsInLightShadow
{
    float4 Position : POSITION0;
    //    float3 Normal : NORMAL0;  // uncommented this will run on gl but not on dx.
    float2 TexureCoordinateA : TEXCOORD0;
};

Well it seems im finding a lot of weird bugs with using the rendertargetcube.

Here is one that really is weird.
The depth buffer is recording weird depths along objects it looks cool but these are visible error shadows on the textures of the objects. The actual visual shadow error on the objects themselves is really terrible looking.

looking at what the cube see’s for depths i get this. you can litterally see the bad shadow depths.

Im guessing this is a precision error why and how i have no idea

What exactly are we looking at here? Are you outputting depth from the main camera with some RGB conversion?