public void RemoveLayer( 
   Layer layer
)

{
    //Get the group layers first
    IReadOnlyList<GroupLayer> groupLayers = map.Layers.OfType<GroupLayer>().ToList();
    //Iterate and remove the layers within the group layers that are unchecked.
    //Note: Run within a QueuedTask
    foreach (var groupLyr in groupLayers)
    {
        //Get layers that not visible within the group
        var layers = groupLyr.Layers.Where(l => l.IsVisible == false).ToList();
        //Remove all the layers that are not visible within the group
        map.RemoveLayers(layers);
    }

    //Group Layers that are empty and are unchecked
    foreach (var group in groupLayers)
    {
        if (group.Layers.Count == 0 && group.IsVisible == false) //No layers in the group
        {
            //remove the group
            map.RemoveLayer(group);
        }
    }

    //Get Layers that are NOT Group layers and are unchecked
    var notAGroupAndUnCheckedLayers = map.Layers.Where(l => !(l is GroupLayer) && l.IsVisible == false).ToList();
    //Remove all the non group layers that are not visible
    map.RemoveLayers(notAGroupAndUnCheckedLayers);
}

{
    //Get the group layers
    IReadOnlyList<GroupLayer> groupLayers = map.Layers.OfType<GroupLayer>().ToList();
    //Note: Run within a QueuedTask
    foreach (var group in groupLayers)
    {
        if (group.Layers.Count == 0) //No layers in the group
        {
            //remove the group
            map.RemoveLayer(group);
        }
    }
}