Changeset 7085 for branches/RegressionBenchmarks/HeuristicLab.Tests

Timestamp:

11/28/11 13:47:28 (12 years ago)

Author:

sforsten

Message:

#1669: branch has been merged with the trunk in revision 7081 and methods in RegressionBenchmark have been renamed.

Location:

branches/RegressionBenchmarks

Files:

• . (modified) (2 props)
• HeuristicLab.Tests (modified) (1 prop)
• HeuristicLab.Tests/HeuristicLab-3.3/SamplesTest.cs (modified) (2 diffs)
• HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/FullTreeCreatorTest.cs (modified) (2 diffs)
• HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/GrowTreeCreatorTest.cs (modified) (2 diffs)
• HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/SubtreeCrossoverTest.cs (modified) (1 diff)
• HeuristicLab.Tests/HeuristicLab.Persistence-3.3/UseCases.cs (modified) (1 diff)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/StatisticCalculatorsTest.cs (modified) (3 diffs)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/TableFileParserTest.cs (modified) (2 diffs)
• HeuristicLab.Tests/HeuristicLab.Tests.csproj (modified) (5 diffs)

branches/RegressionBenchmarks

@@ -4 +4 @@
 *.suo
 *.vsp
+Doxygen
 Google.ProtocolBuffers-0.9.1.dll
 HeuristicLab 3.3.5.1.ReSharper.user
 HeuristicLab 3.3.6.0.ReSharper.user
 HeuristicLab.4.5.resharper.user
+HeuristicLab.ExtLibs.6.0.ReSharper.user
 HeuristicLab.resharper.user
 ProtoGen.exe
@@ -16 +18 @@
 bin
 protoc.exe
-HeuristicLab.ExtLibs.6.0.ReSharper.user
-Doxygen

@@ -4 +4 @@
 *.suo
 *.vsp
+Doxygen
 Google.ProtocolBuffers-0.9.1.dll
 HeuristicLab 3.3.5.1.ReSharper.user
 HeuristicLab 3.3.6.0.ReSharper.user
 HeuristicLab.4.5.resharper.user
+HeuristicLab.ExtLibs.6.0.ReSharper.user
 HeuristicLab.resharper.user
 ProtoGen.exe
@@ -16 +18 @@
 bin
 protoc.exe
-HeuristicLab.ExtLibs.6.0.ReSharper.user
-Doxygen

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab-3.3/SamplesTest.cs

@@ -280 +280 @@
       // configure grammar
       var grammar = new TypeCoherentExpressionGrammar();
-      grammar.Symbols.OfType<Sine>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Cosine>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Tangent>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<IfThenElse>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<GreaterThan>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<LessThan>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<And>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Or>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Not>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<TimeLag>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Integral>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Derivative>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<LaggedVariable>().Single().InitialFrequency = 0.0;
+      grammar.ConfigureAsDefaultRegressionGrammar();
       grammar.Symbols.OfType<VariableCondition>().Single().InitialFrequency = 0.0;
       var varSymbol = grammar.Symbols.OfType<Variable>().Where(x => !(x is LaggedVariable)).Single();
@@ -393 +381 @@
       // configure grammar
       var grammar = new TypeCoherentExpressionGrammar();
-      grammar.Symbols.OfType<Sine>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Cosine>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Tangent>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Power>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Root>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<TimeLag>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Integral>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<Derivative>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<LaggedVariable>().Single().InitialFrequency = 0.0;
-      grammar.Symbols.OfType<VariableCondition>().Single().InitialFrequency = 0.0;
+      grammar.ConfigureAsDefaultClassificationGrammar();
+      grammar.Symbols.OfType<VariableCondition>().Single().Enabled = false;
       var varSymbol = grammar.Symbols.OfType<Variable>().Where(x => !(x is LaggedVariable)).Single();
       varSymbol.WeightMu = 1.0;

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/FullTreeCreatorTest.cs

@@ -31 +31 @@
   public class FullTreeCreatorTest {
     private const int POPULATION_SIZE = 10000;
-    private const int MAX_TREE_DEPTH = 10;
+    private const int MAX_TREE_DEPTH = 9 ;
+    private const int MAX_TREE_LENGTH = 256;
     private TestContext testContextInstance;
 
@@ -55 +56 @@
       stopwatch.Start();
       for (int i = 0; i != POPULATION_SIZE; i++) {
-        randomTrees.Add(FullTreeCreator.Create(random, grammar, MAX_TREE_DEPTH));
+        randomTrees.Add(FullTreeCreator.Create(random, grammar, MAX_TREE_LENGTH, MAX_TREE_DEPTH));
       }
       stopwatch.Stop();

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/GrowTreeCreatorTest.cs

@@ -32 +32 @@
     private const int POPULATION_SIZE = 10000;
     private const int MAX_TREE_DEPTH = 10;
+    private const int MAX_TREE_LENGTH = 50;
     private TestContext testContextInstance;
 
@@ -55 +56 @@
       stopwatch.Start();
       for (int i = 0; i != POPULATION_SIZE; i++) {
-        randomTrees.Add(GrowTreeCreator.Create(random, grammar, MAX_TREE_DEPTH));
+        randomTrees.Add(GrowTreeCreator.Create(random, grammar, MAX_TREE_LENGTH, MAX_TREE_DEPTH));
       }
       stopwatch.Stop();

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4/SubtreeCrossoverTest.cs

@@ -82 +82 @@
         );
 
-      Assert.IsTrue(Math.Round(1000.0 / (msPerCrossoverEvent)) > 2500); // must achieve more than 1500 x-overs/s
+      Assert.IsTrue(Math.Round(1000.0 / (msPerCrossoverEvent)) > 2000); // must achieve more than 2000 x-overs/s
     }
   }

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Persistence-3.3/UseCases.cs

@@ -528 +528 @@
     }
 
-    [TestMethod]
     public void CodingConventions() {
       List<string> lowerCaseMethodNames = new List<string>();

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/StatisticCalculatorsTest.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -121 +122 @@
       for (int c1 = 0; c1 < cols; c1++) {
         double c1Scale = random.NextDouble() * 1E7;
-        double c2Scale = 1.0;
         IEnumerable<double> x = from rows in Enumerable.Range(0, n)
                                 select testData[rows, c1] * c1Scale;
@@ -140 +140 @@
       }
     }
+
+    [TestMethod]
+    public void CalculateDirectionalSymmetryTest() {
+      // delta: +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
+      var original = new double[]
+                       {
+                         0, 
+                         0.01, 
+                         1.01, 
+                         1, 
+                         -1, 
+                         -1.01, 
+                         -2.01, 
+                         -2, 
+                         0
+                       };
+      // delta to original(t-1): +1, +0, -1, -0, -1, +0.01, +0.01, +2
+      var estimated = new double[]
+                        {
+                          -1, 
+                          1, 
+                          0.01,
+                          0.01,
+                          1,
+                          -1,
+                          -1.02,
+                          -2.02,
+                          0
+                        };
+
+      // one-step forecast
+      var startValues = original;
+      var actualContinuations = from x in original.Skip(1)
+                                select Enumerable.Repeat(x, 1);
+      var predictedContinuations = from x in estimated.Skip(1)
+                                   select Enumerable.Repeat(x, 1);
+      double expected = 0.5;  // half of the predicted deltas are correct
+      OnlineCalculatorError errorState;
+      double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
+      Assert.AreEqual(expected, actual, 1E-9);
+    }
+
+    [TestMethod]
+    public void CalculateMultiStepDirectionalSymmetryTest() {
+      // delta: +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
+      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 };
+      {
+        var estimated = new double[][]
+                          {
+                            new double[] {0.01, 1.01, 1, -1, -1.01},
+                            new double[] {1.01, 1, -1, -1.01, -2.01},
+                            new double[] {1, -1, -1.01, -2.01, -2},
+                            new double[] {-1, -1.01, -2.01, -2, 0}
+                          };
+
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+        double expected = 1; // predictions are 100% correct
+        OnlineCalculatorError errorState;
+        double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations,
+                                                                      predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-9);
+      }
+      {
+        // only the direction is relevant
+        var estimated = new double[][]
+                          {
+                            new double[] {0.01, 0.01, 0.01, -0.01, -0.01},  // start=0, original deltas: 0.01, 1.01, 1.00, -1.00, -1.01
+                            new double[] {0.02, 0.02, 0.00, 0.00, 0.00}, // start=0.01, original deltas: 1.00, 0.90, -1.01, -1.02, -2.02,
+                            new double[] { 1.00, 1.00, 1.00, 1.00, 1.00}, // start=1.01, original deltas: -0.01, -2.01, -2.02, -3.02, -3.01
+                            new double[] { 0.90, 0.90, 0.90, 0.90, 0.90}  // start=1, original deltas: -2.00, -0.01, -3.01, -3.00, -1.00
+                          };
+
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+        double expected = 1; // half of the predicted deltas are correct
+        OnlineCalculatorError errorState;
+        double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations,
+                                                                      predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-9);
+      }
+      {
+        // also check incorrectly predicted directions
+        var estimated = new double[][]
+                          {
+                            new double[] {0.01, 0.01, 0.01, +0.01, +0.01},  // start=0, original deltas: 0.01, 1.01, 1.00, -1.00, -1.01
+                            new double[] {0.02, 0.00, 0.02, 0.00, 0.00}, // start=0.01, original deltas: 1.00, 0.90, -1.01, -1.02, -2.02,
+                            new double[] { 1.02, 1.00, 1.02, 1.00, 1.02}, // start=1.01, original deltas: -0.01, -2.01, -2.02, -3.02, -3.01
+                            new double[] { 0.90, 0.90, 0.90, 0.90, 0.90}  // start=1, original deltas: -2.00, -0.01, -3.01, -3.00, -1.00
+                          };
+
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+        double expected = (20 - 7) / 20.0; // half of the predicted deltas are correct
+        OnlineCalculatorError errorState;
+        double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations,
+                                                                      predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-9);
+      }
+    }
+
+
+    [TestMethod]
+    public void CalculateWeightedDirectionalSymmetryTest() {
+      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 }; // +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
+      var estimated = new double[] { 1, 2, 2, 1, 1, 0, 0.01, 0.02, 2.02 }; // delta to original: +2, +1.99, -0.01, 0, +1, -1.02, +2.01, +4.02
+      // one-step forecast
+      var startValues = original;
+      var actualContinuations = from x in original.Skip(1)
+                                select Enumerable.Repeat(x, 1);
+      var predictedContinuations = from x in estimated.Skip(1)
+                                   select Enumerable.Repeat(x, 1);
+      // absolute errors = 1.99, 0.99, 0, 2, 1.01, 2.02, 2.02, 2.02     
+      // sum of absolute errors for correctly predicted deltas = 2.97
+      // sum of absolute errors for incorrectly predicted deltas = 3.03 
+      double expected = 5.03 / 7.02;
+      OnlineCalculatorError errorState;
+      double actual = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
+      Assert.AreEqual(expected, actual, 1E-9);
+    }
+
+    [TestMethod]
+    public void CalculateTheilsUTest() {
+      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 };
+      var estimated = new double[] { 1, 1.01, 0.01, 2, 0, -0.01, -1.01, -3, 1 };
+      // one-step forecast
+      var startValues = original;
+      var actualContinuations = from x in original.Skip(1)
+                                select Enumerable.Repeat(x, 1);
+      var predictedContinuations = from x in estimated.Skip(1)
+                                   select Enumerable.Repeat(x, 1);
+      // Sum of squared errors of model y(t+1) = y(t) = 10.0004
+      // Sum of squared errors of predicted values = 8  
+      double expected = Math.Sqrt(8 / 10.0004);
+      OnlineCalculatorError errorState;
+      double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
+      Assert.AreEqual(expected, actual, 1E-9);
+    }
+
+    [TestMethod]
+    public void CalculateMultiStepTheilsUTest() {
+      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 };
+      {
+        // prefect prediction
+        var estimated = new double[][]
+                          {
+                            new double[] {0.01, 1.01, 1, -1, -1.01},
+                            new double[] {1.01, 1, -1, -1.01, -2.01},
+                            new double[] {1, -1, -1.01, -2.01, -2},
+                            new double[] {-1, -1.01, -2.01, -2, 0}
+                          };
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+
+        double expected = 0;
+        OnlineCalculatorError errorState;
+        double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations,
+                                                                   predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-9);
+      }
+      {
+        // naive prediction
+        var estimated = new double[][]
+                          {
+                            new double[] {0, 0, 0, 0, 0},
+                            new double[] {0.01, 0.01, 0.01, 0.01, 0.01},
+                            new double[] {1.01, 1.01, 1.01, 1.01, 1.01},
+                            new double[] {1, 1, 1, 1, 1}
+                          };
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+
+        double expected = 1;
+        OnlineCalculatorError errorState;
+        double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations,
+                                                                   predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-9);
+      }
+      {
+        // realistic prediction
+        var estimated = new double[][]
+                          {
+                            new double[] {0.005, 0.5, 0.5, -0.5, -0.5},    // start = 0
+                            new double[] {0.60, 0.5, -0.5, -0.5, -1},   // start = 0.01
+                            new double[] {-0.005, 0, 0, -0.5, -1},     // start = 1.01
+                            new double[] {-0, 0, -0.5, -1, 0.5}      // start = 1
+                          };
+        // 5-step forecast
+        var startValues = original.Take(4);
+        var actualContinuations = from i in Enumerable.Range(1, original.Count() - 5)
+                                  select original.Skip(i).Take(5);
+        var predictedContinuations = estimated;
+
+        double expected = 0.47558387;
+        OnlineCalculatorError errorState;
+        double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations,
+                                                                   predictedContinuations, out errorState);
+        Assert.AreEqual(expected, actual, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    public void CalculateAccuracyTest() {
+      var original = new double[] { 1, 1, 0, 0 };
+      var estimated = new double[] { 1, 0, 1, 0 };
+      double expected = 0.5;
+      OnlineCalculatorError errorState;
+      double actual = OnlineAccuracyCalculator.Calculate(original, estimated, out errorState);
+      Assert.AreEqual(expected, actual, 1E-9);
+    }
+
+    [TestMethod]
+    public void CalculateMeanAbsolutePercentageErrorTest() {
+      var original = new double[] { 1, 2, 3, 1, 5 };
+      var estimated = new double[] { 2, 1, 3, 1, 0 };
+      double expected = 0.5;
+      OnlineCalculatorError errorState;
+      double actual = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original, estimated, out errorState);
+      Assert.AreEqual(expected, actual, 1E-9);
+      Assert.AreEqual(OnlineCalculatorError.None, errorState);
+
+      // if the original contains zero values the result is not defined
+      var original2 = new double[] { 1, 2, 0, 0, 0 };
+      OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original2, estimated, out errorState);
+      Assert.AreEqual(OnlineCalculatorError.InvalidValueAdded, errorState);
+    }
   }
 }

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/TableFileParserTest.cs

@@ -185 +185 @@
 
     [TestMethod]
+    public void ParseEnglishCSVWithoutCommasWithoutSpace() {
+      string tempFileName = Path.GetTempFileName();
+      WriteToFile(tempFileName,
+@"0,0,0,3
+0,0,0,0
+0,0,0,0
+0,0,0,0
+0,0,0,0
+0,0,0,0");
+      TableFileParser parser = new TableFileParser();
+      try {
+        parser.Parse(tempFileName);
+        Assert.AreEqual(6, parser.Rows);
+        Assert.AreEqual(4, parser.Columns);
+        Assert.AreEqual((double)parser.Values[3][0], 3);
+      }
+      finally {
+        File.Delete(tempFileName);
+      }
+    }
+
+    [TestMethod]
     public void ParseEnglishCSVWithoutCommasWithNames() {
       string tempFileName = Path.GetTempFileName();
@@ -195 +217 @@
 0, 0, 0, 0
 0, 0, 0, 0");
+      TableFileParser parser = new TableFileParser();
+      try {
+        parser.Parse(tempFileName);
+        Assert.AreEqual(6, parser.Rows);
+        Assert.AreEqual(4, parser.Columns);
+        Assert.AreEqual((double)parser.Values[3][0], 3);
+      }
+      finally {
+        File.Delete(tempFileName);
+      }
+    }
+
+    [TestMethod]
+    public void ParseEnglishCSVWithoutCommasWithoutSpacesWithNames() {
+      string tempFileName = Path.GetTempFileName();
+      WriteToFile(tempFileName,
+@"x01,x02,x03,x04
+0,0,0,3
+0,0,0,0
+0,0,0,0
+0,0,0,0
+0,0,0,0
+0,0,0,0");
       TableFileParser parser = new TableFileParser();
       try {

branches/RegressionBenchmarks/HeuristicLab.Tests/HeuristicLab.Tests.csproj

@@ -104 +104 @@
     <Reference Include="ALGLIB-3.1.0">
       <HintPath>..\bin\ALGLIB-3.1.0.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Calendar.DayView">
       <HintPath>..\bin\Calendar.DayView.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="Google.ProtocolBuffers-0.9.1, Version=0.9.0.0, Culture=neutral, PublicKeyToken=17b3b1f090c3ea48, processorArchitecture=AMD64">
       <HintPath>..\bin\Google.ProtocolBuffers-0.9.1.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.ALGLIB-3.1.0">
       <HintPath>..\bin\HeuristicLab.ALGLIB-3.1.0.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.LibSVM-1.6.3">
       <HintPath>..\bin\HeuristicLab.LibSVM-1.6.3.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.log4net-1.2.10.0">
       <HintPath>..\bin\HeuristicLab.log4net-1.2.10.0.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.Netron-3.0.2672.12446">
       <HintPath>..\bin\HeuristicLab.Netron-3.0.2672.12446.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.ProtobufCS-0.9.1">
       <HintPath>..\bin\HeuristicLab.ProtobufCS-0.9.1.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.SharpDevelop-3.1.1.5327">
       <HintPath>..\bin\HeuristicLab.SharpDevelop-3.1.1.5327.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="HeuristicLab.WinFormsUI-2.3.1">
       <HintPath>..\bin\HeuristicLab.WinFormsUI-2.3.1.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.NRefactory, Version=3.1.1.5327, Culture=neutral, PublicKeyToken=efe927acf176eea2, processorArchitecture=MSIL">
       <HintPath>..\bin\ICSharpCode.NRefactory.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpDevelop.Dom, Version=3.1.1.5327, Culture=neutral, PublicKeyToken=f829da5c02be14ee, processorArchitecture=MSIL">
       <HintPath>..\bin\ICSharpCode.SharpDevelop.Dom.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.SharpZipLib, Version=0.85.4.369, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
       <HintPath>..\bin\ICSharpCode.SharpZipLib.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="ICSharpCode.TextEditor, Version=3.1.1.5327, Culture=neutral, PublicKeyToken=4d61825e8dd49f1a, processorArchitecture=MSIL">
       <HintPath>..\bin\ICSharpCode.TextEditor.dll</HintPath>
-    </Reference>
-    <Reference Include="LibSVM-1.6.3, Version=1.6.3.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=AMD64">
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="LibSVM-1.6.3, Version=1.6.3.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <HintPath>..\bin\LibSVM-1.6.3.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="log4net, Version=1.2.10.0, Culture=neutral, PublicKeyToken=1b44e1d426115821, processorArchitecture=MSIL">
       <HintPath>..\bin\log4net.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />
-    <Reference Include="Netron.Diagramming.Core-3.0.2672.12446, Version=3.0.2672.12446, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=AMD64">
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="Netron.Diagramming.Core-3.0.2672.12446, Version=3.0.2672.12446, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <HintPath>..\bin\Netron.Diagramming.Core-3.0.2672.12446.dll</HintPath>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System" />
@@ -160 +179 @@
     <Reference Include="System.Drawing" />
     <Reference Include="System.Runtime.Serialization" />
-    <Reference Include="WeifenLuo.WinFormsUI.Docking-2.3.1, Version=2.3.1.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=AMD64">
+    <Reference Include="WeifenLuo.WinFormsUI.Docking-2.3.1, Version=2.3.1.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <HintPath>..\bin\WeifenLuo.WinFormsUI.Docking-2.3.1.dll</HintPath>
+      <Private>True</Private>
     </Reference>
   </ItemGroup>
@@ -570 +590 @@
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Benchmarks.Views\3.3\HeuristicLab.Algorithms.Benchmarks.Views-3.3.csproj">
+      <Project>{3C906172-E044-4DF0-B4FD-AA21192D5D3E}</Project>
+      <Name>HeuristicLab.Algorithms.Benchmarks.Views-3.3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Benchmarks\3.3\HeuristicLab.Algorithms.Benchmarks-3.3.csproj">
+      <Project>{068C70A7-8CAF-4AB6-8721-4564D62292BC}</Project>
+      <Name>HeuristicLab.Algorithms.Benchmarks-3.3</Name>
+    </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Algorithms.DataAnalysis.Views\3.4\HeuristicLab.Algorithms.DataAnalysis.Views-3.4.csproj">
       <Project>{A2330DB0-413D-4424-B4E6-591118E2B9F8}</Project>
@@ -626 +654 @@
       <Name>HeuristicLab.Clients.Common-3.3</Name>
     </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Clients.Hive.Administrator\3.3\HeuristicLab.Clients.Hive.Administrator-3.3.csproj">
+      <Project>{5211F1AD-BBA2-4C62-8749-DCA0A25CF5D8}</Project>
+      <Name>HeuristicLab.Clients.Hive.Administrator-3.3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Clients.Hive.JobManager\3.3\HeuristicLab.Clients.Hive.JobManager-3.3.csproj">
+      <Project>{DBFD7205-61AF-40EF-8776-3052DEF91CF3}</Project>
+      <Name>HeuristicLab.Clients.Hive.JobManager-3.3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Clients.Hive.Views\3.3\HeuristicLab.Clients.Hive.Views-3.3.csproj">
+      <Project>{E1D6C801-892A-406A-B606-F158E36DD3C3}</Project>
+      <Name>HeuristicLab.Clients.Hive.Views-3.3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Clients.Hive\3.3\HeuristicLab.Clients.Hive-3.3.csproj">
+      <Project>{B5EF1E5A-9F3D-40B9-B4B0-30AADF2E2CEB}</Project>
+      <Name>HeuristicLab.Clients.Hive-3.3</Name>
+    </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.CodeEditor\3.3\HeuristicLab.CodeEditor-3.3.csproj">
       <Project>{489CFE09-FDF7-4C89-BAB5-BD09CADD61AD}</Project>
@@ -689 +733 @@
       <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
       <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Hive\3.3\HeuristicLab.Hive-3.3.csproj">
+      <Project>{F98A1740-9AC9-4D36-A582-6A2D0D06978D}</Project>
+      <Name>HeuristicLab.Hive-3.3</Name>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.MainForm.WindowsForms\3.3\HeuristicLab.MainForm.WindowsForms-3.3.csproj">

@@ -4 +4 @@
 *.suo
 *.vsp
+Doxygen
 Google.ProtocolBuffers-0.9.1.dll
 HeuristicLab 3.3.5.1.ReSharper.user
 HeuristicLab 3.3.6.0.ReSharper.user
 HeuristicLab.4.5.resharper.user
+HeuristicLab.ExtLibs.6.0.ReSharper.user
 HeuristicLab.resharper.user
 ProtoGen.exe
@@ -16 +18 @@
 bin
 protoc.exe
-HeuristicLab.ExtLibs.6.0.ReSharper.user
-Doxygen