source: trunk/sources/HeuristicLab.Tests/HeuristicLab.Scripting-3.3/GridSearchScriptTest.cs @ 11466

Last change on this file since 11466 was 11466, checked in by bburlacu, 6 years ago

#2211: Updated grid search start page scripts and unit tests, which now reside in HeuristicLab.Scripting and belong to test category "Scripting".

File size: 6.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2014 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.CodeDom.Compiler;
24using System.IO;
25using System.Linq;
26using System.Reflection;
27using HeuristicLab.Common;
28using HeuristicLab.Core;
29using HeuristicLab.Optimizer;
30using HeuristicLab.Persistence.Default.Xml;
31using HeuristicLab.Problems.DataAnalysis;
32using HeuristicLab.Scripting;
33using Microsoft.VisualStudio.TestTools.UnitTesting;
34
35namespace HeuristicLab.Tests {
36  [TestClass]
37  public class GridSearchScriptTest {
38    private const string PathPrefix = "HeuristicLab.Optimizer.Documents.";
39    private const string PathSuffix = ".hl";
40    private const string SvmClassificationScriptName = "GridSearch_SVM_Classification";
41    private const string SvmRegressionScriptName = "GridSearch_SVM_Regression";
42    private const string RandomForestRegressionScriptName = "GridSearch_RF_Regression";
43    private const string RandomForestClassificationScriptName = "GridSearch_RF_Classification";
44    private const string SamplesDirectory = SamplesUtils.Directory;
45
46    [ClassInitialize]
47    public static void MyClassInitialize(TestContext testContext) {
48      PluginLoader.Assemblies.Any();
49      if (!Directory.Exists(SamplesDirectory))
50        Directory.CreateDirectory(SamplesDirectory);
51    }
52
53    [TestMethod]
54    [TestCategory("Scripting")]
55    public void RunRandomForestRegressionScriptTest() {
56      var assembly = new StartPage().GetType().Assembly;
57      const string name = PathPrefix + RandomForestRegressionScriptName + PathSuffix;
58      var script = (CSharpScript)LoadSample(name, assembly);
59
60      try {
61        script.Compile();
62      }
63      catch {
64        if (script.CompileErrors.HasErrors) {
65          ShowCompilationResults(script);
66          throw new Exception("Compilation failed.");
67        } else {
68          Console.WriteLine("Compilation succeeded.");
69        }
70      }
71      finally {
72        script.Execute();
73        var vs = script.VariableStore;
74        var solution = (IRegressionSolution)vs["demo_bestSolution"];
75        Assert.IsTrue(solution.TrainingRSquared.IsAlmost(1));
76      }
77    }
78
79    [TestMethod]
80    [TestCategory("Scripting")]
81    public void RunRandomForestClassificationScriptTest() {
82      var assembly = new StartPage().GetType().Assembly;
83      const string name = PathPrefix + RandomForestClassificationScriptName + PathSuffix;
84      var script = (CSharpScript)LoadSample(name, assembly);
85
86      try {
87        script.Compile();
88      }
89      catch {
90        if (script.CompileErrors.HasErrors) {
91          ShowCompilationResults(script);
92          throw new Exception("Compilation failed.");
93        } else {
94          Console.WriteLine("Compilation succeeded.");
95        }
96      }
97      finally {
98        script.Execute();
99        var vs = script.VariableStore;
100        var solution = (IClassificationSolution)vs["demo_bestSolution"];
101        Assert.IsTrue(solution.TrainingAccuracy.IsAlmost(1) && solution.TestAccuracy.IsAlmost(0.953125));
102      }
103    }
104
105    [TestMethod]
106    [TestCategory("Scripting")]
107    public void RunSvmRegressionScriptTest() {
108      var assembly = new StartPage().GetType().Assembly;
109      const string name = PathPrefix + SvmRegressionScriptName + PathSuffix;
110      var script = (CSharpScript)LoadSample(name, assembly);
111
112      try {
113        script.Compile();
114      }
115      catch {
116        if (script.CompileErrors.HasErrors) {
117          ShowCompilationResults(script);
118          throw new Exception("Compilation failed.");
119        } else {
120          Console.WriteLine("Compilation succeeded.");
121        }
122      }
123      finally {
124        script.Execute();
125        var vs = script.VariableStore;
126        var solution = (IRegressionSolution)vs["demo_bestSolution"];
127        Assert.IsTrue(solution.TrainingRSquared.IsAlmost(0.066221959224331) && solution.TestRSquared.IsAlmost(0.0794407638195883));
128      }
129    }
130
131    [TestMethod]
132    [TestCategory("Scripting")]
133    public void RunSvmClassificationScriptTest() {
134      var assembly = new StartPage().GetType().Assembly;
135      const string name = PathPrefix + SvmClassificationScriptName + PathSuffix;
136      var script = (CSharpScript)LoadSample(name, assembly);
137
138      try {
139        script.Compile();
140      }
141      catch {
142        if (script.CompileErrors.HasErrors) {
143          ShowCompilationResults(script);
144          throw new Exception("Compilation failed.");
145        } else {
146          Console.WriteLine("Compilation succeeded.");
147        }
148      }
149      finally {
150        script.Execute();
151        var vs = script.VariableStore;
152        var solution = (IClassificationSolution)vs["demo_bestSolution"];
153        Assert.IsTrue(solution.TrainingAccuracy.IsAlmost(0.817472698907956) && solution.TestAccuracy.IsAlmost(0.809375));
154      }
155    }
156
157    #region Helpers
158    private static void ShowCompilationResults(Script script) {
159      if (script.CompileErrors.Count == 0) return;
160      var msgs = script.CompileErrors.OfType<CompilerError>()
161                                      .OrderBy(x => x.IsWarning)
162                                      .ThenBy(x => x.Line)
163                                      .ThenBy(x => x.Column);
164      foreach (var m in msgs) {
165        Console.WriteLine(m);
166      }
167    }
168
169    private INamedItem LoadSample(string name, Assembly assembly) {
170      string path = Path.GetTempFileName();
171      INamedItem item = null;
172      try {
173        using (var stream = assembly.GetManifestResourceStream(name)) {
174          WriteStreamToTempFile(stream, path); // create a file in a temporary folder (persistence cannot load these files directly from the stream)
175          item = XmlParser.Deserialize<INamedItem>(path);
176        }
177      }
178      catch (Exception) {
179      }
180      finally {
181        if (File.Exists(path)) {
182          File.Delete(path); // make sure we remove the temporary file
183        }
184      }
185      return item;
186    }
187
188    private void WriteStreamToTempFile(Stream stream, string path) {
189      using (FileStream output = new FileStream(path, FileMode.Create, FileAccess.Write)) {
190        stream.CopyTo(output);
191      }
192    }
193    #endregion
194  }
195}
Note: See TracBrowser for help on using the repository browser.