Sorry, I did not want to post code on first place.
Basically I have 4 different kind of scenes. The player is able to travel from one place to another. Because in every scene other assets are used, I want to clear everything up before moving to the other scene. I am using static classes to share resources between gameobjects. Those classes work as follows:
public Texture2D this[Common common]
        {
            get
            {
                if (!sprites.ContainsKey(common))
                    sprites.Add(common, ResourceManager.SceneResources.Load<Texture2D>(GetPathToResource(common)));
                else if (sprites.ContainsKey(common) && sprites[common].IsDisposed)
                {
                    sprites.Remove(common);
                    return this[common];
                }
                return sprites[common];
            }
        }
The main ResourceManager which all the other static classes access:
public static class ResourceManager
{
    private static event Action sceneResourcesNoLongerAvailable;
    private const string ROOT_DIRECTORY = "Content";
    static ResourceManager()
    {
        CreateApplicationResourceManager();
        CreateSceneResourceManager();
    }
    public static ContentManager ApplicationResources { get; private set; }
    public static ContentManager SceneResources { get; private set; }
    public static event Action SceneResourcesNoLongerAvailable
    {
        add { sceneResourcesNoLongerAvailable += value; }
        remove { sceneResourcesNoLongerAvailable -= value; }
    }
    private static void CreateApplicationResourceManager()
    {
        if (ApplicationResources != null)
            return;
        
        ApplicationResources = new ContentManager(Game.Singleton.Content.ServiceProvider, ROOT_DIRECTORY);
    }
    public static void CreateSceneResourceManager()
    {
        if (SceneResources != null)
            CleanSceneResourceManager();
        SceneResources = new ContentManager(Game.Singleton.Content.ServiceProvider, ROOT_DIRECTORY);
    }
    public static void CleanSceneResourceManager()
    {
        SceneResources.Unload();
        SceneResources.Dispose();
        SceneResources = null;
        sceneResourcesNoLongerAvailable?.Invoke();
    }
}
If the player decides to go to another level, following will be called
private void DisposeLevel(Level level)
        {
            /* "Beautifully dispose all the level stuff */
            UserInterfaceManager.Singleton.RemoveAndDisposeAllControls();
            level.BackgroundLevel.Dispose();
            level.EntityManager.Dispose();
            /* Force to dispose the rest we may have missed because of my shit coding skills */
            ResourceManager.CleanSceneResourceManager();
            InputManager.Singleton.RemoveAllListeners();
            Camera.ResetPosition();
            Camera.Clean();
            /* Remove all the components */
            UserInterfaceManager.Singleton.Dispose();
            InputManager.Singleton.Dispose();
            Game.Singleton.Components.Remove(UserInterfaceManager.Singleton);
            Game.Singleton.Components.Remove(InputManager.Singleton);
            /* Call GC */
            Debug.WriteLine(GC.GetTotalMemory(true));
            GC.Collect();
            GC.WaitForPendingFinalizers();
            Debug.WriteLine(GC.GetTotalMemory(true));
        }
        private void CreateLevel(Level level)
        {
            ResourceManager.CreateSceneResourceManager();
            Game.Singleton.Components.Add(new UserInterfaceManager(Game.Singleton));
            Game.Singleton.Components.Add(new InputManager(Game.Singleton));
            level.InitializeLevel(ResourceManager.SceneResources);
            level.FinalizeLevel();
        }
Also the old screen will be removed from the screenManager
    if (currentLevel != null) ScreenManager.RemoveScreen(currentLevel);
    var workThread = Task.Factory.StartNew(() =>
    {
        if(currentLevel != null)
            DisposeLevel(currentLevel);
        CreateLevel(nextLevel);
        ScreenManager.AddScreen(nextLevel);
        RemoveScreen(this);
    });
EDIT
It seems like after
        SceneResources.Unload();
        SceneResources.Dispose();
        SceneResources = null;
Nearly no memory has been released. I put a GC.GetTotalMemory before and after those lines, and the result is as following.
--------------------
MEM 20MB 21493KB
MEM 20MB 20994KB
--------------------
--------------------
MEM 32MB 33220KB
MEM 31MB 32721KB
--------------------
--------------------
MEM 43MB 44940KB
MEM 43MB 44441KB
--------------------
--------------------
MEM 55MB 56666KB
MEM 54MB 56167KB
--------------------
--------------------
MEM 66MB 68387KB
MEM 66MB 67888KB
--------------------