Testing configurations like this can give you a false sense of security, because configurations can be changed post-deployment. This is unlike the rest of the code that you're unit testing, which would require a recompilation in order for the changes to take effect.
If you still want to have a test for this situation, though, your test will need to manually load the web.config file and parse it for the expected value. (You can't use the ConfigurationManager, since your test project uses its own app.config instead of the web.config you're targeting.)
For example:
using System.Xml.Linq;
using System.Xml.XPath;
[TestClass]
public class WebConfigTest
{
    /// <summary>
    /// in Web.config, debug mode must be set to 'false'
    /// </summary>
    [TestMethod]
    public void TestDebugFalse()
    {
        var path        = @"Path to web.config";
        var config      = XDocument.Load(path);
        var compilation = config.XPathSelectElement("/configuration/system.web/compilation");
        if(compilation != null)
        {
            var debug = compilation.Attribute("debug");
            if(debug != null)
            {
                Assert.AreEqual("false", debug.Value);
            }
        }
    }
}
Edit: Joe mentioned in the comments that you can alternatively use the WebConfigurationManager to get this information, without the XML parsing. For example:
[TestClass]
public class WebConfigTest
{
    /// <summary>
    /// in Web.config, debug mode must be set to 'false'
    /// </summary>
    [TestMethod]
    public void TestDebugFalse()
    {
        var path                    = @"Path to web.config";
        var webConfig               = new System.IO.FileInfo(path);
        var virtualDirectoryMapping = new System.Web.Configuration.VirtualDirectoryMapping(webConfig.DirectoryName, true, webConfig.Name);
        var fileMap                 = new System.Web.Configuration.WebConfigurationFileMap();
        fileMap.VirtualDirectories.Add("/", virtualDirectoryMapping);
        var configuration           = System.Web.Configuration.WebConfigurationManager.OpenMappedWebConfiguration(fileMap, "/");
        var compilation             = configuration.GetSection("system.web/compilation") as System.Web.Configuration.CompilationSection;
        if(compilation != null)
        {
            Assert.AreEqual(false, compilation.Debug);
        }
    }
}