DrawInstancedPrimitives ignores instance buffer data

Hello to everybody,
this is my first post, I have been following the forum for a while finding a lot of answers to my numerous questions about Monogame and 3D graphics is general, but now I really have a problem I cannot seem to be able to solve :frowning:

In short: I need to draw a lot of copies of the same procedurally generated vertex/index buffers, so I decided to resort to hardware instancing, but I canā€™t make it work properly, I only get one copy of the model. It looks like my instance buffer contents are just ignored.

I couldnā€™t find any working tutorial on this topic, so instead of copy-pasting my own non-working game code begging for help I decided to make a small demo program that shows the problem so it will stay as a reference for future searches: it just draws an array of cubes switching between DrawIndexedPrimitives and DrawInstancedPrimitives as you press the mouse button.
When using DrawIndexedPrimitives the array of cubes is rendered, while when using DrawInstancedPrimitives just one cube is rendered.

I run it on Windows 10 in a DirectX monogame project

using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using Microsoft.Xna.Framework.Input;
using System.Runtime.InteropServices;
using System.Linq;

namespace MonogameHardwareInstancingDemo
{

  public class Game1 : Microsoft.Xna.Framework.Game
  {


    GraphicsDeviceManager graphics;
    VertexBuffer vertexBuffer;
    IndexBuffer indexBuffer;
    BasicEffect basicEffect;

    Matrix view = Matrix.CreateLookAt(new Vector3(0, -10, 10), new Vector3(0, 0, 0), new Vector3(0, 1, 0));
    Matrix projection = Matrix.CreatePerspectiveFieldOfView(MathHelper.ToRadians(45), 800f / 480f, 0.01f, 100f);

    RasterizerState rasterizerState = new RasterizerState() { CullMode = CullMode.None };

    Matrix[] instances = Enumerable.Range(-10, 21).SelectMany((x) => Enumerable.Range(-10, 21).Select((y) => Matrix.CreateTranslation(new Vector3(2 * x, 2 * y, 0)))).ToArray();


    public Game1()
    {
      graphics = new GraphicsDeviceManager(this);
      Content.RootDirectory = "Content";
    }

    protected override void Initialize()
    {
      base.Initialize();
    }

    protected override void LoadContent()
    {
      basicEffect = new BasicEffect(GraphicsDevice);
      basicEffect.EnableDefaultLighting();
      basicEffect.View = view;
      basicEffect.Projection = projection;
      
      basicEffect.VertexColorEnabled = true;

      _buildCube(Color.LightGray, out VertexPositionColorNormal[] vertices, out short[] indices);

      vertexBuffer = new VertexBuffer(GraphicsDevice, typeof(VertexPositionColorNormal), vertices.Length, BufferUsage.WriteOnly);
      vertexBuffer.SetData(vertices);

      indexBuffer = new IndexBuffer(graphics.GraphicsDevice, typeof(short), indices.Length, BufferUsage.WriteOnly);
      indexBuffer.SetData(indices, 0, indices.Length);
    }

    protected override void UnloadContent()
    {
    }

    protected override void Update(GameTime gameTime)
    {
      // Allows the game to exit
      if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
        this.Exit();

      base.Update(gameTime);
    }

    protected override void Draw(GameTime gameTime)
    {
      GraphicsDevice.Clear(Color.CornflowerBlue);
      GraphicsDevice.RasterizerState = rasterizerState;
      

      if (Mouse.GetState().LeftButton == ButtonState.Pressed)
        DrawIndexedPrimitives();
     else
        DrawInstancedPrimitives();

      base.Draw(gameTime);
    }


    private void DrawIndexedPrimitives()
    {

      basicEffect.DiffuseColor = Color.Red.ToVector3();
      GraphicsDevice.SetVertexBuffer(vertexBuffer);
      GraphicsDevice.Indices = indexBuffer;

      foreach (var world in instances) {
        basicEffect.World = world;

        foreach (EffectPass pass in basicEffect.CurrentTechnique.Passes) {
          pass.Apply();
          GraphicsDevice.DrawIndexedPrimitives(PrimitiveType.TriangleList, 0, 0, indexBuffer.IndexCount / 3);
        }
      }
    }


    private DynamicVertexBuffer _instanceVertexBuffer;

    private void DrawInstancedPrimitives()
    {

      basicEffect.DiffuseColor = Color.Blue.ToVector3();
      basicEffect.World = Matrix.Identity;
    
      if ((_instanceVertexBuffer == null) || (instances.Count() > _instanceVertexBuffer.VertexCount)) {
        if (_instanceVertexBuffer != null)
          _instanceVertexBuffer.Dispose();

        _instanceVertexBuffer = new DynamicVertexBuffer(GraphicsDevice, InstanceVertex.VertexDeclaration, instances.Count(), BufferUsage.WriteOnly);
      }

      _instanceVertexBuffer.SetData(instances, 0, instances.Count(), SetDataOptions.Discard);

      GraphicsDevice.SetVertexBuffers(
          new VertexBufferBinding(vertexBuffer, 0, 0),
          new VertexBufferBinding(_instanceVertexBuffer, 0, 1)
      );

      GraphicsDevice.Indices = indexBuffer;

      foreach (EffectPass pass in basicEffect.CurrentTechnique.Passes) {
        pass.Apply();
        GraphicsDevice.DrawInstancedPrimitives(PrimitiveType.TriangleList, 0, 0, indexBuffer.IndexCount / 3, instances.Count());
      }
    }

    private static void _buildCube(Color c, out VertexPositionColorNormal[] vertices, out short[] indices)
    {

      vertices = new VertexPositionColorNormal[] {
          new VertexPositionColorNormal(new Vector3(0,0,0),c, -Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(0,1,0),c, -Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(1,1,0),c, -Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(1,0,0),c, -Vector3.UnitZ),

          new VertexPositionColorNormal(new Vector3(0,0,1),c, Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(0,1,1),c, Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(1,1,1),c, Vector3.UnitZ),
          new VertexPositionColorNormal(new Vector3(1,0,1),c, Vector3.UnitZ),

          new VertexPositionColorNormal(new Vector3(0,0,0),c, -Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(0,0,1),c, -Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(0,1,1),c, -Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(0,1,0),c, -Vector3.UnitX),

          new VertexPositionColorNormal(new Vector3(1,0,0),c, Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(1,0,1),c, Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(1,1,1),c, Vector3.UnitX),
          new VertexPositionColorNormal(new Vector3(1,1,0),c, Vector3.UnitX),

          new VertexPositionColorNormal(new Vector3(0,0,0),c, -Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(1,0,0),c, -Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(1,0,1),c, -Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(0,0,1),c, -Vector3.UnitY),

          new VertexPositionColorNormal(new Vector3(0,1,0),c, Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(1,1,0),c, Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(1,1,1),c, Vector3.UnitY),
          new VertexPositionColorNormal(new Vector3(0,1,1),c, Vector3.UnitY),
    };

      indices = new short[] {0,1,2,   0,2,3,
                              4,6,5,   4,7,6,
                              8,9,10,  8,10,11,
                              12,13,14, 12,14,15,
                              16,17,18,  16,18,19,
                              20,23,22, 20,22,21};
    }

  }


  [StructLayout(LayoutKind.Sequential, Pack = 1)]
  public struct VertexPositionColorNormal : IVertexType
  {
    public Vector3 Position;
    public Vector3 Normal;
    public Color Color;

    public readonly static VertexDeclaration VertexDeclaration
        = new VertexDeclaration(
            new VertexElement(0, VertexElementFormat.Vector3, VertexElementUsage.Position, 0),
            new VertexElement(12, VertexElementFormat.Vector3, VertexElementUsage.Normal, 0),
            new VertexElement(24, VertexElementFormat.Color, VertexElementUsage.Color, 0)
            );

    public VertexPositionColorNormal(Vector3 pos, Color c, Vector3 n)
    {
      Position = pos;
      Color = c;
      Normal = n;
    }

    VertexDeclaration IVertexType.VertexDeclaration
    {
      get { return VertexDeclaration; }
    }
  }



  [StructLayout(LayoutKind.Sequential, Pack = 1)]
  public struct InstanceVertex : IVertexType
  {
   
    public readonly static VertexDeclaration VertexDeclaration
        = new VertexDeclaration(
            new VertexElement(0, VertexElementFormat.Vector4, VertexElementUsage.Color, 1),
            new VertexElement(16, VertexElementFormat.Vector4, VertexElementUsage.Color, 2),
            new VertexElement(32, VertexElementFormat.Vector4, VertexElementUsage.Color, 3),
            new VertexElement(48, VertexElementFormat.Vector4, VertexElementUsage.Color, 4)
            );

    VertexDeclaration IVertexType.VertexDeclaration
    {
      get { return VertexDeclaration; }
    }
  }

}

Iā€™m pretty sure BasicEffect doesnā€™t support instancing. You need a custom shader.

I was starting to suspect that ā€¦ but I hoped there was a different solution.

Defining a custom shader that supports all the neat features of BasicEffect plus hardware instancing sounds like a lot of work, especially since I have zero shader programming experience.

I may as well approach the problem from another perspective: do not draw many copies of the same mesh but rather combine the copies into a single larger mesh.
Itā€™s all procedural meshes after all, so I can add a ā€œmergingā€ step to my code.