Skip to content

Commit

Permalink
Test Runner improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
jp2masa committed Dec 31, 2017
1 parent 0105de6 commit b231f66
Show file tree
Hide file tree
Showing 12 changed files with 169 additions and 188 deletions.
81 changes: 21 additions & 60 deletions Tests/Cosmos.TestRunner.Core/DefaultEngineConfiguration.cs
Original file line number Diff line number Diff line change
@@ -1,72 +1,33 @@
using Cosmos.Build.Common;
using System;
using System.Reflection;
using System;
using System.Collections.Generic;

using Cosmos.Build.Common;

namespace Cosmos.TestRunner.Core
{
public static class DefaultEngineConfiguration
public class DefaultEngineConfiguration : IEngineConfiguration
{
public static void Apply(Engine engine)
{
if (engine == null)
{
throw new ArgumentNullException("engine");
}

// Sets the time before an error is registered. For example if set to 60 then if a kernel runs for more than 60 seconds then
// that kernel will be marked as a failure and terminated
engine.AllowedSecondsInKernel = 6000;

// If you want to test only specific platforms, add them to the list, like next line. By default, all platforms are run.
engine.RunTargets.Add(RunTargetEnum.Bochs);
//engine.RunTargets.Add(RunTargetEnum.VMware);
//engine.RunTargets.Add(RunTargetEnum.HyperV);

// If you're working on the compiler (or other lower parts), you can choose to run the compiler in process
// one thing to keep in mind though, is that this only works with 1 kernel at a time!
engine.DebugIL2CPU = false;
engine.TraceAssembliesLevel = TraceAssemblies.User;
//engine.EnableStackCorruptionChecks = false;
engine.KernelPkg = "X86";

engine.EnableStackCorruptionChecks = true;
engine.StackCorruptionChecksLevel = StackCorruptionDetectionLevel.AllInstructions;
public virtual int AllowedSecondsInKernel => 6000;

//engine.RunWithGDB = true;
//engine.StartBochsDebugGui = true;

// Select kernels to be tested by adding them to the engine
foreach (var xType in TestKernelSets.GetKernelTypesToRun())
public virtual IEnumerable<RunTargetEnum> RunTargets
{
get
{
engine.AddKernel(xType.GetTypeInfo().Assembly.Location);
yield return RunTargetEnum.Bochs;
//yield return RunTargetEnum.VMware;
//yield return RunTargetEnum.HyperV;
}
}

//engine.AddKernel(typeof(Cosmos.Compiler.Tests.TypeSystem.Kernel).Assembly.Location);
//engine.AddKernel(typeof(VGACompilerCrash.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.Bcl.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.SingleEchoTest.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.SimpleWriteLine.Kernel.Kernel).Assembly.Location);
//engine.AddKernel(typeof(SimpleStructsAndArraysTest.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.Exceptions.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.MethodTests.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Kernel.Tests.Fat.Kernel).Assembly.Location);

// Known bugs, therefore disabled for now:
//engine.AddKernel(typeof(BoxingTests.Kernel).Assembly.Location);
//engine.AddKernel(typeof(Cosmos.Compiler.Tests.MultidimensionalArrays.Kernel).Assembly.Location);

// Experimental stuff:
public virtual bool RunWithGDB => false;
public virtual bool StartBochsDebugGUI => false;

// end of known bugs
public virtual bool DebugIL2CPU => false;
public virtual string KernelPkg => String.Empty;
public virtual TraceAssemblies TraceAssembliesLevel => TraceAssemblies.User;
public virtual bool EnableStackCorruptionChecks => true;
public virtual StackCorruptionDetectionLevel StackCorruptionChecksLevel => StackCorruptionDetectionLevel.AllInstructions;

// double check: this check is in the engine, but lets put it here as well
if (engine.DebugIL2CPU)
{
if (engine.KernelsToRun.Count > 1 || engine.RunTargets.Count == 0 || engine.RunTargets.Count > 1)
{
throw new InvalidOperationException("Can only run 1 kernel if IL2CPU is ran in-process!");
}
}
}
public virtual IEnumerable<Type> KernelTypesToRun => TestKernelSets.GetStableKernelTypes();
}
}
20 changes: 10 additions & 10 deletions Tests/Cosmos.TestRunner.Core/Engine.Helpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -189,20 +189,20 @@ private void RunTheRingMaster(string kernelFileName)

private void RunIL2CPU(string kernelFileName, string outputFile)
{
References = new List<string>() { kernelFileName };
var xReferences = new List<string>() { kernelFileName };

if (KernelPkg == "X86")
{
References.Add(Assembly.Load(new AssemblyName("Cosmos.CPU_Plugs")).Location);
References.Add(Assembly.Load(new AssemblyName("Cosmos.CPU_Asm")).Location);
References.Add(Assembly.Load(new AssemblyName("Cosmos.Plugs.TapRoot")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.CPU_Plugs")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.CPU_Asm")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.Plugs.TapRoot")).Location);
}
else
{
References.Add(Assembly.Load(new AssemblyName("Cosmos.Core_Plugs")).Location);
References.Add(Assembly.Load(new AssemblyName("Cosmos.Core_Asm")).Location);
References.Add(Assembly.Load(new AssemblyName("Cosmos.System2_Plugs")).Location);
References.Add(Assembly.Load(new AssemblyName("Cosmos.Debug.Kernel.Plugs.Asm")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.Core_Plugs")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.Core_Asm")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.System2_Plugs")).Location);
xReferences.Add(Assembly.Load(new AssemblyName("Cosmos.Debug.Kernel.Plugs.Asm")).Location);
}

var xArgs = new List<string>
Expand All @@ -220,7 +220,7 @@ private void RunIL2CPU(string kernelFileName, string outputFile)
"IgnoreDebugStubAttribute:False"
};

xArgs.AddRange(References.Select(aReference => "References:" + aReference));
xArgs.AddRange(xReferences.Select(aReference => "References:" + aReference));

bool xUsingUserkit = false;
string xIL2CPUPath = Path.Combine(FindCosmosRoot(), "..", "IL2CPU", "source", "IL2CPU.Compiler");
Expand All @@ -238,7 +238,7 @@ private void RunIL2CPU(string kernelFileName, string outputFile)
{
if (DebugIL2CPU)
{
if (KernelsToRun.Count > 1)
if (KernelsToRun.Count() > 1)
{
throw new Exception("Cannot run multiple kernels with in-process compilation!");
}
Expand Down
4 changes: 2 additions & 2 deletions Tests/Cosmos.TestRunner.Core/Engine.Run.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ namespace Cosmos.TestRunner.Core
{
partial class Engine
{
public int AllowedSecondsInKernel = 30;
public List<RunTargetEnum> RunTargets = new List<RunTargetEnum>();
protected int AllowedSecondsInKernel => mConfiguration.AllowedSecondsInKernel;
protected IEnumerable<RunTargetEnum> RunTargets => mConfiguration.RunTargets;

private bool ExecuteKernel(string assemblyFileName, RunConfiguration configuration)
{
Expand Down
45 changes: 21 additions & 24 deletions Tests/Cosmos.TestRunner.Core/Engine.cs
Original file line number Diff line number Diff line change
@@ -1,49 +1,46 @@
using Cosmos.Build.Common;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

using Cosmos.Build.Common;

namespace Cosmos.TestRunner.Core
{
public partial class Engine
{
// configuration: in process eases debugging, but means certain errors (like stack overflow) kill the test runner.
public bool DebugIL2CPU = false;
public bool RunWithGDB = false;
public bool StartBochsDebugGui = false;
public bool EnableStackCorruptionChecks = true;
public string KernelPkg = "";
public TraceAssemblies TraceAssembliesLevel = TraceAssemblies.User;
public StackCorruptionDetectionLevel StackCorruptionChecksLevel = StackCorruptionDetectionLevel.MethodFooters;
public List<string> References = new List<string>();
public List<string> AdditionalSearchDirs = new List<string>();
public List<string> AdditionalReferences = new List<string>();
protected bool DebugIL2CPU => mConfiguration.DebugIL2CPU;
protected string KernelPkg => mConfiguration.KernelPkg;
protected TraceAssemblies TraceAssembliesLevel => mConfiguration.TraceAssembliesLevel;
protected bool EnableStackCorruptionChecks => mConfiguration.EnableStackCorruptionChecks;
protected StackCorruptionDetectionLevel StackCorruptionChecksLevel => mConfiguration.StackCorruptionChecksLevel;

public List<string> KernelsToRun { get; } = new List<string>();
protected bool RunWithGDB => mConfiguration.RunWithGDB;
protected bool StartBochsDebugGui => mConfiguration.StartBochsDebugGUI;

public void AddKernel(string assemblyFile)
{
if (!File.Exists(assemblyFile))
{
throw new FileNotFoundException("Kernel file not found!", assemblyFile);
}
KernelsToRun.Add(assemblyFile);
}
public IEnumerable<Type> KernelsToRun => mConfiguration.KernelTypesToRun;

private IEngineConfiguration mConfiguration;
private string mBaseWorkingDirectory;

public OutputHandlerBasic OutputHandler;

public Engine(IEngineConfiguration aEngineConfiguration)
{
mConfiguration = aEngineConfiguration;
}

public bool Execute()
{
if (OutputHandler == null)
{
throw new InvalidOperationException("No OutputHandler set!");
}

if (RunTargets.Count == 0)
if (!RunTargets.Any())
{
RunTargets.AddRange((RunTargetEnum[])Enum.GetValues(typeof(RunTargetEnum)));
throw new InvalidOperationException("No run targets were specified!");
}

OutputHandler.ExecutionStart();
Expand All @@ -55,7 +52,7 @@ public bool Execute()
OutputHandler.RunConfigurationStart(xConfig);
try
{
foreach (var xAssemblyFile in KernelsToRun)
foreach (var xKernelType in KernelsToRun)
{
mBaseWorkingDirectory = Path.Combine(Path.GetDirectoryName(typeof(Engine).Assembly.Location), "WorkingDirectory");
if (Directory.Exists(mBaseWorkingDirectory))
Expand All @@ -64,7 +61,7 @@ public bool Execute()
}
Directory.CreateDirectory(mBaseWorkingDirectory);

xResult &= ExecuteKernel(xAssemblyFile, xConfig);
xResult &= ExecuteKernel(xKernelType.Assembly.Location, xConfig);
}
}
catch (Exception e)
Expand Down
23 changes: 23 additions & 0 deletions Tests/Cosmos.TestRunner.Core/IEngineConfiguration.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;

using Cosmos.Build.Common;

namespace Cosmos.TestRunner.Core
{
public interface IEngineConfiguration
{
int AllowedSecondsInKernel { get; }
IEnumerable<RunTargetEnum> RunTargets { get; }
bool RunWithGDB { get; }
bool StartBochsDebugGUI { get; }

bool DebugIL2CPU { get; }
string KernelPkg { get; }
TraceAssemblies TraceAssembliesLevel { get; }
bool EnableStackCorruptionChecks { get; }
StackCorruptionDetectionLevel StackCorruptionChecksLevel { get; }

IEnumerable<Type> KernelTypesToRun { get; }
}
}
4 changes: 1 addition & 3 deletions Tests/Cosmos.TestRunner.UI/MainWindowHandler.TestEngine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ public void RunTestEngine()

private void TestEngineThreadMain()
{
var xEngine = new Engine();

DefaultEngineConfiguration.Apply(xEngine);
var xEngine = new Engine(new DefaultEngineConfiguration());

var xOutputXml = new OutputHandlerXml();
xEngine.OutputHandler = new MultiplexingOutputHandler(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.3.0" />
<PackageReference Include="NUnit" Version="3.7.1" />
<PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
<PackageReference Include="NUnit" Version="3.9.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.9.0" />
</ItemGroup>

<ItemGroup>
Expand Down
38 changes: 38 additions & 0 deletions Tests/Cosmos.TestRunner.UnitTest/EngineConfiguration.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;

using Cosmos.Build.Common;
using Cosmos.TestRunner.Core;

namespace Cosmos.TestRunner.UnitTest
{
internal class EngineConfiguration : DefaultEngineConfiguration
{
private Type mKernelType;

public EngineConfiguration(Type aKernelType)
{
mKernelType = aKernelType;
}

public override int AllowedSecondsInKernel => 1200;

public override IEnumerable<RunTargetEnum> RunTargets
{
get
{
yield return RunTargetEnum.Bochs;
}
}

public override StackCorruptionDetectionLevel StackCorruptionChecksLevel => StackCorruptionDetectionLevel.MethodFooters;

public override IEnumerable<Type> KernelTypesToRun
{
get
{
yield return mKernelType;
}
}
}
}
49 changes: 49 additions & 0 deletions Tests/Cosmos.TestRunner.UnitTest/KernelTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
using System.IO;

using NUnit.Framework;

using Cosmos.TestRunner.Core;

namespace Cosmos.TestRunner.UnitTest
{
using Assert = NUnit.Framework.Assert;

[TestFixture]
public class KernelTests
{
private static IEnumerable<Type> KernelsToRun => TestKernelSets.GetStableKernelTypes();

[TestCaseSource(nameof(KernelsToRun))]
public void TestKernel(Type aKernelType)
{
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(typeof(KernelTests).Assembly.Location));

var xEngine = new Engine(new EngineConfiguration(aKernelType));
xEngine.OutputHandler = new TestOutputHandler();

Assert.IsTrue(xEngine.Execute());
}
catch (AssertionException)
{
throw;
}
catch (Exception e)
{
Console.WriteLine("Exception occurred: " + e.ToString());
Assert.Fail();
}
}

private class TestOutputHandler : OutputHandlerFullTextBase
{
protected override void Log(string message)
{
TestContext.WriteLine(String.Concat(DateTime.Now.ToString("hh:mm:ss.ffffff "), new string(' ', mLogLevel * 2), message));
}
}
}
}
Loading

0 comments on commit b231f66

Please sign in to comment.