Changeset 17984 for branches

Timestamp:

05/29/21 15:45:50 (3 years ago)

Author:

gkronber

Message:

#3106 updated implementation based on the reply by Moscato

Location:

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• ContinuedFractionRegression/Agent.cs (added)
• ContinuedFractionRegression/Algorithm.cs (modified) (24 diffs)
• ContinuedFractionRegression/ContinuedFraction.cs (modified) (1 diff)
• ContinuedFractionRegression/NelderMeadSimplex.cs (modified) (1 diff)
• ContinuedFractionRegression/Transformation.cs (modified) (1 diff)
• ContinuedFractionRegression/Vector.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (1 diff)

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/ContinuedFractionRegression/Algorithm.cs

@@ -4 +4 @@
 using System.Threading;
 using HEAL.Attic;
+using HeuristicLab.Analysis;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -15 +16 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis.ContinuedFractionRegression {
+  /// <summary>
+  /// Implementation of Continuous Fraction Regression (CFR) as described in 
+  /// Pablo Moscato, Haoyuan Sun, Mohammad Nazmul Haque,
+  /// Analytic Continued Fractions for Regression: A Memetic Algorithm Approach,
+  /// Expert Systems with Applications, Volume 179, 2021, 115018, ISSN 0957-4174,
+  /// https://doi.org/10.1016/j.eswa.2021.115018.
+  /// </summary>
   [Item("Continuous Fraction Regression (CFR)", "TODO")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 999)]
@@ -21 +29 @@
     private const string MutationRateParameterName = "MutationRate";
     private const string DepthParameterName = "Depth";
-    private const string NumGenerationsParameterName = "Depth";
+    private const string NumGenerationsParameterName = "NumGenerations";
+    private const string StagnationGenerationsParameterName = "StagnationGenerations";
+    private const string LocalSearchIterationsParameterName = "LocalSearchIterations";
+    private const string LocalSearchRestartsParameterName = "LocalSearchRestarts";
+    private const string LocalSearchToleranceParameterName = "LocalSearchTolerance";
+    private const string DeltaParameterName = "Delta";
+    private const string ScaleDataParameterName = "ScaleData";
+
 
     #region parameters
@@ -39 +54 @@
       set { NumGenerationsParameter.Value.Value = value; }
     }
+    public IFixedValueParameter<IntValue> StagnationGenerationsParameter => (IFixedValueParameter<IntValue>)Parameters[StagnationGenerationsParameterName];
+    public int StagnationGenerations {
+      get { return StagnationGenerationsParameter.Value.Value; }
+      set { StagnationGenerationsParameter.Value.Value = value; }
+    }
+    public IFixedValueParameter<IntValue> LocalSearchIterationsParameter => (IFixedValueParameter<IntValue>)Parameters[LocalSearchIterationsParameterName];
+    public int LocalSearchIterations {
+      get { return LocalSearchIterationsParameter.Value.Value; }
+      set { LocalSearchIterationsParameter.Value.Value = value; }
+    }
+    public IFixedValueParameter<IntValue> LocalSearchRestartsParameter => (IFixedValueParameter<IntValue>)Parameters[LocalSearchRestartsParameterName];
+    public int LocalSearchRestarts {
+      get { return LocalSearchRestartsParameter.Value.Value; }
+      set { LocalSearchRestartsParameter.Value.Value = value; }
+    }
+    public IFixedValueParameter<DoubleValue> LocalSearchToleranceParameter => (IFixedValueParameter<DoubleValue>)Parameters[LocalSearchToleranceParameterName];
+    public double LocalSearchTolerance {
+      get { return LocalSearchToleranceParameter.Value.Value; }
+      set { LocalSearchToleranceParameter.Value.Value = value; }
+    }
+    public IFixedValueParameter<PercentValue> DeltaParameter => (IFixedValueParameter<PercentValue>)Parameters[DeltaParameterName];
+    public double Delta {
+      get { return DeltaParameter.Value.Value; }
+      set { DeltaParameter.Value.Value = value; }
+    }
+    public IFixedValueParameter<BoolValue> ScaleDataParameter => (IFixedValueParameter<BoolValue>)Parameters[ScaleDataParameterName];
+    public bool ScaleData {
+      get { return ScaleDataParameter.Value.Value; }
+      set { ScaleDataParameter.Value.Value = value; }
+    }
     #endregion
 
@@ -51 +96 @@
     // default ctor
     public Algorithm() : base() {
+      Problem = new RegressionProblem();
       Parameters.Add(new FixedValueParameter<PercentValue>(MutationRateParameterName, "Mutation rate (default 10%)", new PercentValue(0.1)));
       Parameters.Add(new FixedValueParameter<IntValue>(DepthParameterName, "Depth of the continued fraction representation (default 6)", new IntValue(6)));
       Parameters.Add(new FixedValueParameter<IntValue>(NumGenerationsParameterName, "The maximum number of generations (default 200)", new IntValue(200)));
+      Parameters.Add(new FixedValueParameter<IntValue>(StagnationGenerationsParameterName, "Number of generations after which the population is re-initialized (default value 5)", new IntValue(5)));
+      Parameters.Add(new FixedValueParameter<IntValue>(LocalSearchIterationsParameterName, "Number of iterations for local search (simplex) (default value 250)", new IntValue(250)));
+      Parameters.Add(new FixedValueParameter<IntValue>(LocalSearchRestartsParameterName, "Number of restarts for local search (default value 4)", new IntValue(4)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(LocalSearchToleranceParameterName, "The tolerance value for local search (simplex) (default value: 1e-3)", new DoubleValue(1e-3)));
+      Parameters.Add(new FixedValueParameter<PercentValue>(DeltaParameterName, "The relative weight for the number of variables term in the fitness function (default value: 10%)", new PercentValue(0.1)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(ScaleDataParameterName, "Turns on/off scaling of input variable values to the range [0 .. 1] (default: false)", new BoolValue(false)));
     }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      throw new NotImplementedException();
+      return new Algorithm(this, cloner);
     }
 
     protected override void Run(CancellationToken cancellationToken) {
       var problemData = Problem.ProblemData;
-
-      var x = problemData.Dataset.ToArray(problemData.AllowedInputVariables.Concat(new[] { problemData.TargetVariable }),
-        problemData.TrainingIndices);
-      var nVars = x.GetLength(1) - 1;
+      double[,] xy;
+      if (ScaleData) {
+        // Scale data to range 0 .. 1
+        // 
+        // Scaling was not used for the experiments in the paper. Statement by the authors: "We did not pre-process the data."
+        var transformations = new List<Transformation<double>>();
+        foreach (var input in problemData.AllowedInputVariables) {
+          var values = problemData.Dataset.GetDoubleValues(input, problemData.TrainingIndices);
+          var linTransformation = new LinearTransformation(problemData.AllowedInputVariables);
+          var min = values.Min();
+          var max = values.Max();
+          var range = max - min;
+          linTransformation.Addend = -min / range;
+          linTransformation.Multiplier = 1.0 / range;
+          transformations.Add(linTransformation);
+        }
+        // do not scale the target 
+        transformations.Add(new LinearTransformation(problemData.AllowedInputVariables) { Addend = 0.0, Multiplier = 1.0 });
+        xy = problemData.Dataset.ToArray(problemData.AllowedInputVariables.Concat(new[] { problemData.TargetVariable }),
+          transformations,
+          problemData.TrainingIndices);
+      } else {
+        // no transformation
+        xy = problemData.Dataset.ToArray(problemData.AllowedInputVariables.Concat(new[] { problemData.TargetVariable }),
+          problemData.TrainingIndices);
+      }
+      var nVars = xy.GetLength(1) - 1;
       var seed = new System.Random().Next();
       var rand = new MersenneTwister((uint)seed);
-      CFRAlgorithm(nVars, Depth, MutationRate, x, out var best, out var bestObj, rand, NumGenerations, stagnatingGens: 5, cancellationToken);
+      CFRAlgorithm(nVars, Depth, MutationRate, xy, out var bestObj, rand, NumGenerations, StagnationGenerations,
+        Delta,
+        LocalSearchIterations, LocalSearchRestarts, LocalSearchTolerance, cancellationToken);
     }
 
     private void CFRAlgorithm(int nVars, int depth, double mutationRate, double[,] trainingData,
-      out ContinuedFraction best, out double bestObj,
-      IRandom rand, int numGen, int stagnatingGens,
+      out double bestObj,
+      IRandom rand, int numGen, int stagnatingGens, double evalDelta,
+      int localSearchIterations, int localSearchRestarts, double localSearchTolerance,
       CancellationToken cancellationToken) {
       /* Algorithm 1 */
       /* Generate initial population by a randomized algorithm */
       var pop = InitialPopulation(nVars, depth, rand, trainingData);
-      best = pop.pocket;
       bestObj = pop.pocketObjValue;
-      var bestObjGen = 0;
+      // the best value since the last reset
+      var episodeBestObj = pop.pocketObjValue;
+      var episodeBestObjGen = 0;
       for (int gen = 1; gen <= numGen && !cancellationToken.IsCancellationRequested; gen++) {
         /* mutate each current solution in the population */
@@ -87 +166 @@
         var pop_r = RecombinePopulation(pop_mu, rand, nVars);
 
+        // "We ran the Local Search after Mutation and recombination operations. We executed the local-search only on the Current solutions."
+        // "We executed the MaintainInvariant() in the following steps:
+        // - After generating the initial population
+        // - after resetting the root
+        // - after executing the local-search on the whole population.
+        // We updated the pocket/ current automatically after mutation and recombination operation."
+
         /* local search optimization of current solutions */
         foreach (var agent in pop_r.IterateLevels()) {
-          LocalSearchSimplex(agent.current, ref agent.currentObjValue, trainingData, rand); // CHECK paper states that pocket might also be optimized. Unclear how / when invariants are maintained.
-        }
-
-        foreach (var agent in pop_r.IteratePostOrder()) agent.MaintainInvariant(); // CHECK deviates from Alg1 in paper 
+          LocalSearchSimplex(localSearchIterations, localSearchRestarts, localSearchTolerance, evalDelta, agent.current, ref agent.currentObjValue, trainingData, rand);
+        }
+        foreach (var agent in pop_r.IteratePostOrder()) agent.MaintainInvariant(); // post-order to make sure that the root contains the best model 
+
+
+        // for detecting stagnation we track the best objective value since the last reset 
+        // and reset if this does not change for stagnatingGens
+        if (gen > episodeBestObjGen + stagnatingGens) {
+          Reset(pop_r, nVars, depth, rand, trainingData);
+          episodeBestObj = double.MaxValue;
+        }
+        if (episodeBestObj > pop_r.pocketObjValue) {
+          episodeBestObjGen = gen; // wait at least stagnatingGens until resetting again
+          episodeBestObj = pop_r.pocketObjValue;
+        }
 
         /* replace old population with evolved population */
@@ -98 +195 @@
 
         /* keep track of the best solution */
-        if (bestObj > pop.pocketObjValue) { // CHECK: comparison obviously wrong in the paper
-          best = pop.pocket;
+        if (bestObj > pop.pocketObjValue) {
           bestObj = pop.pocketObjValue;
-          bestObjGen = gen;
-          // Results.AddOrUpdateResult("MSE (best)", new DoubleValue(bestObj));
-          // Results.AddOrUpdateResult("Solution", CreateSymbolicRegressionSolution(best, Problem.ProblemData));
-        }
-
-
-        if (gen > bestObjGen + stagnatingGens) {
-          bestObjGen = gen; // CHECK: unspecified in the paper: wait at least stagnatingGens until resetting again
-          Reset(pop, nVars, depth, rand, trainingData);
-          // InitialPopulation(nVars, depth, rand, trainingData); CHECK reset is not specified in the paper
-        }
+          Results.AddOrUpdateResult("MSE (best)", new DoubleValue(bestObj));
+          Results.AddOrUpdateResult("Solution", CreateSymbolicRegressionSolution(pop.pocket, trainingData, Problem.ProblemData.AllowedInputVariables.ToArray(), Problem.ProblemData.TargetVariable));
+        }
+
+        #region visualization and debugging
+        DataTable qualities;
+        int i = 0;
+        if (Results.TryGetValue("Qualities", out var qualitiesResult)) {
+          qualities = (DataTable)qualitiesResult.Value;
+        } else {
+          qualities = new DataTable("Qualities", "Qualities");
+          i = 0;
+          foreach (var node in pop.IterateLevels()) {
+            qualities.Rows.Add(new DataRow($"Quality {i} pocket", "Quality of pocket"));
+            qualities.Rows.Add(new DataRow($"Quality {i} current", "Quality of current"));
+            i++;
+          }
+          Results.AddOrUpdateResult("Qualities", qualities);
+        }
+        i = 0;
+        foreach (var node in pop.IterateLevels()) {
+          qualities.Rows[$"Quality {i} pocket"].Values.Add(node.pocketObjValue);
+          qualities.Rows[$"Quality {i} current"].Values.Add(node.currentObjValue);
+          i++;
+        }
+        #endregion
       }
     }
@@ -128 +239 @@
       }
 
+      // Statement by the authors: "Yes, we use post-order traversal here"
       foreach (var agent in pop.IteratePostOrder()) {
         agent.current = new ContinuedFraction(nVars, depth, rand);
         agent.pocket = new ContinuedFraction(nVars, depth, rand);
 
-        agent.currentObjValue = Evaluate(agent.current, trainingData);
-        agent.pocketObjValue = Evaluate(agent.pocket, trainingData);
+        agent.currentObjValue = Evaluate(agent.current, trainingData, Delta);
+        agent.pocketObjValue = Evaluate(agent.pocket, trainingData, Delta);
 
         /* within each agent, the pocket solution always holds the better value of guiding
@@ -143 +255 @@
     }
 
-    // TODO: reset is not described in the paper
+    // Reset is not described in detail in the paper.
+    // Statement by the authors: "We only replaced the pocket solution of the root with
+    // a randomly generated solution. Then we execute the maintain-invariant process.
+    // It does not initialize the solutions in the entire population."
     private void Reset(Agent root, int nVars, int depth, IRandom rand, double[,] trainingData) {
       root.pocket = new ContinuedFraction(nVars, depth, rand);
       root.current = new ContinuedFraction(nVars, depth, rand);
 
-      root.currentObjValue = Evaluate(root.current, trainingData);
-      root.pocketObjValue = Evaluate(root.pocket, trainingData);
-
-      /* within each agent, the pocket solution always holds the better value of guiding
-       * function than its current solution
-       */
-      root.MaintainInvariant();
+      root.currentObjValue = Evaluate(root.current, trainingData, Delta);
+      root.pocketObjValue = Evaluate(root.pocket, trainingData, Delta);
+
+      foreach (var agent in root.IteratePreOrder()) { agent.MaintainInvariant(); } // Here we push the newly created model down the hierarchy. 
     }
 
@@ -167 +279 @@
         var s3 = pop.children[2];
 
-        // CHECK Deviates from paper (recombine all models in the current pop before updating the population)
-        var l_current = Recombine(l.pocket, s1.current, SelectRandomOp(rand), rand, nVars);
-        var s3_current = Recombine(s3.pocket, l.current, SelectRandomOp(rand), rand, nVars);
-        var s1_current = Recombine(s1.pocket, s2.current, SelectRandomOp(rand), rand, nVars);
-        var s2_current = Recombine(s2.pocket, s3.current, SelectRandomOp(rand), rand, nVars);
+        // Statement by the authors: "we are using recently generated solutions.
+        // For an example, in step 1 we got the new current(l), which is being used in
+        // Step 2 to generate current(s3). The current(s3) from Step 2 is being used at
+        // Step 4. These steps are executed sequentially from 1 to 4. Similarly, in the
+        // recombination of lower-level subpopulations, we will have the new current
+        // (the supporters generated at the previous level) as the leader of the subpopulation.
+        l.current = Recombine(l.pocket, s1.current, SelectRandomOp(rand), rand, nVars);
+        s3.current = Recombine(s3.pocket, l.current, SelectRandomOp(rand), rand, nVars);
+        s1.current = Recombine(s1.pocket, s2.current, SelectRandomOp(rand), rand, nVars);
+        s2.current = Recombine(s2.pocket, s3.current, SelectRandomOp(rand), rand, nVars);
 
         // recombination works from top to bottom
-        // CHECK do we use the new current solutions (s1_current .. s3_current) already in the next levels?
         foreach (var child in pop.children) {
           RecombinePopulation(child, rand, nVars);
         }
 
-        l.current = l_current;
-        s3.current = s3_current;
-        s1.current = s1_current;
-        s2.current = s2_current;
       }
       return pop;
-    }
-
-    private Func<bool[], bool[], bool[]> SelectRandomOp(IRandom rand) {
-      bool[] union(bool[] a, bool[] b) {
-        var res = new bool[a.Length];
-        for (int i = 0; i < a.Length; i++) res[i] = a[i] || b[i];
-        return res;
-      }
-      bool[] intersect(bool[] a, bool[] b) {
-        var res = new bool[a.Length];
-        for (int i = 0; i < a.Length; i++) res[i] = a[i] && b[i];
-        return res;
-      }
-      bool[] symmetricDifference(bool[] a, bool[] b) {
-        var res = new bool[a.Length];
-        for (int i = 0; i < a.Length; i++) res[i] = a[i] ^ b[i];
-        return res;
-      }
-      switch (rand.Next(3)) {
-        case 0: return union;
-        case 1: return intersect;
-        case 2: return symmetricDifference;
-        default: throw new ArgumentException();
-      }
     }
 
@@ -223 +311 @@
         /* recombine coefficient values for variables */
         var coefx = new double[nVars];
-        var varsx = new bool[nVars]; // CHECK: deviates from paper, probably forgotten in the pseudo-code
-        for (int vi = 1; vi < nVars; vi++) {
+        var varsx = new bool[nVars]; // deviates from paper, probably forgotten in the pseudo-code
+        for (int vi = 0; vi < nVars; vi++) {
           if (ch.vars[vi]) {  // CHECK: paper uses featAt()
             if (varsa[vi] && varsb[vi]) {
@@ -244 +332 @@
         ch.h[i].beta = p1.h[i].beta + (rand.NextDouble() * 5 - 1) * (p2.h[i].beta - p1.h[i].beta) / 3.0;
       }
-      /* update current solution and apply local search */
-      // return LocalSearchSimplex(ch, trainingData); // CHECK: Deviates from paper because Alg1 also has LocalSearch after Recombination
+      // return LocalSearchSimplex(ch, trainingData); // The paper has a local search step here. 
+      // The authors have stated that local search is executed after mutation and recombination
+      // for the current solutions.
+      // Local search and MaintainInvariant is called in the main loop (Alg 1)
       return ch;
     }
@@ -253 +343 @@
         if (rand.NextDouble() < mutationRate) {
           if (agent.currentObjValue < 1.2 * agent.pocketObjValue ||
-             agent.currentObjValue > 2 * agent.pocketObjValue)
+              agent.currentObjValue > 2 * agent.pocketObjValue)
             ToggleVariables(agent.current, rand); // major mutation
           else
@@ -301 +391 @@
       var h = cfrac.h[rand.Next(cfrac.h.Length)];
 
-      /* modify the coefficient value*/
+      /* modify the coefficient value */
       if (h.vars[vIdx]) {  // CHECK: paper uses varAt()
         h.coef[vIdx] = 0.0;
@@ -311 +401 @@
     }
 
-    private static double Evaluate(ContinuedFraction cfrac, double[,] trainingData) {
+    private static double Evaluate(ContinuedFraction cfrac, double[,] trainingData, double delta) {
       var dataPoint = new double[trainingData.GetLength(1) - 1];
       var yIdx = trainingData.GetLength(1) - 1;
@@ -324 +414 @@
         sum += res * res;
       }
-      var delta = 0.1;
       return sum / trainingData.GetLength(0) * (1 + delta * cfrac.vars.Count(vi => vi));
     }
@@ -342 +431 @@
     }
 
+
+    private Func<bool[], bool[], bool[]> SelectRandomOp(IRandom rand) {
+      bool[] union(bool[] a, bool[] b) {
+        var res = new bool[a.Length];
+        for (int i = 0; i < a.Length; i++) res[i] = a[i] || b[i];
+        return res;
+      }
+      bool[] intersect(bool[] a, bool[] b) {
+        var res = new bool[a.Length];
+        for (int i = 0; i < a.Length; i++) res[i] = a[i] && b[i];
+        return res;
+      }
+      bool[] symmetricDifference(bool[] a, bool[] b) {
+        var res = new bool[a.Length];
+        for (int i = 0; i < a.Length; i++) res[i] = a[i] ^ b[i];
+        return res;
+      }
+      switch (rand.Next(3)) {
+        case 0: return union;
+        case 1: return intersect;
+        case 2: return symmetricDifference;
+        default: throw new ArgumentException();
+      }
+    }
+
     private static double dot(bool[] filter, double[] x, double[] y) {
       var s = 0.0;
@@ -351 +465 @@
 
 
-    private static void LocalSearchSimplex(ContinuedFraction ch, ref double quality, double[,] trainingData, IRandom rand) {
+    private static void LocalSearchSimplex(int iterations, int restarts, double tolerance, double delta, ContinuedFraction ch, ref double quality, double[,] trainingData, IRandom rand) {
       double uniformPeturbation = 1.0;
-      double tolerance = 1e-3;
-      int maxEvals = 250;
-      int numSearches = 4;
+      int maxEvals = iterations;
+      int numSearches = restarts + 1;
       var numRows = trainingData.GetLength(0);
       int numSelectedRows = numRows / 5; // 20% of the training samples
 
-      quality = Evaluate(ch, trainingData); // get quality with origial coefficients
+      quality = Evaluate(ch, trainingData, delta); // get quality with origial coefficients
 
       double[] origCoeff = ExtractCoeff(ch);
@@ -371 +484 @@
       double objFunc(double[] curCoeff) {
         SetCoeff(ch, curCoeff);
-        return Evaluate(ch, fittingData);
+        return Evaluate(ch, fittingData, delta);
       }
 
@@ -386 +499 @@
         SetCoeff(ch, optimizedCoeff);
 
-        var newQuality = Evaluate(ch, trainingData);
+        var newQuality = Evaluate(ch, trainingData, delta);
 
         if (newQuality < bestQuality) {
@@ -434 +547 @@
     }
 
+    #region build a symbolic expression tree
     Symbol addSy = new Addition();
-    Symbol mulSy = new Multiplication();
     Symbol divSy = new Division();
     Symbol startSy = new StartSymbol();
@@ -442 +555 @@
     Symbol varSy = new Problems.DataAnalysis.Symbolic.Variable();
 
-    private ISymbolicRegressionSolution CreateSymbolicRegressionSolution(ContinuedFraction cfrac, IRegressionProblemData problemData) {
-      var variables = problemData.AllowedInputVariables.ToArray();
+    private ISymbolicRegressionSolution CreateSymbolicRegressionSolution(ContinuedFraction cfrac, double[,] trainingData, string[] variables, string targetVariable) {
       ISymbolicExpressionTreeNode res = null;
       for (int i = cfrac.h.Length - 1; i > 1; i -= 2) {
@@ -469 +581 @@
       start.AddSubtree(h0Term);
 
-      var model = new SymbolicRegressionModel(problemData.TargetVariable, new SymbolicExpressionTree(progRoot), new SymbolicDataAnalysisExpressionTreeBatchInterpreter());
-      var sol = new SymbolicRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+      var model = new SymbolicRegressionModel(targetVariable, new SymbolicExpressionTree(progRoot), new SymbolicDataAnalysisExpressionTreeBatchInterpreter());
+      var ds = new Dataset(variables.Concat(new[] { targetVariable }), trainingData);
+      var problemData = new RegressionProblemData(ds, variables, targetVariable);
+      var sol = new SymbolicRegressionSolution(model, problemData);
       return sol;
     }
@@ -494 +608 @@
     }
   }
-
-  public class Agent {
-    public ContinuedFraction pocket;
-    public double pocketObjValue;
-    public ContinuedFraction current;
-    public double currentObjValue;
-
-    public IList<Agent> children = new List<Agent>();
-
-    public IEnumerable<Agent> IterateLevels() {
-      var agents = new List<Agent>() { this };
-      IterateLevelsRec(this, agents);
-      return agents;
-    }
-    public IEnumerable<Agent> IteratePostOrder() {
-      var agents = new List<Agent>();
-      IteratePostOrderRec(this, agents);
-      return agents;
-    }
-
-    internal void MaintainInvariant() {
-      foreach (var child in children) {
-        MaintainInvariant(parent: this, child);
-      }
-      if (currentObjValue < pocketObjValue) {
-        Swap(ref pocket, ref current);
-        Swap(ref pocketObjValue, ref currentObjValue);
-      }
-    }
-
-
-    private static void MaintainInvariant(Agent parent, Agent child) {
-      if (child.pocketObjValue < parent.pocketObjValue) {
-        Swap(ref child.pocket, ref parent.pocket);
-        Swap(ref child.pocketObjValue, ref parent.pocketObjValue);
-      }
-    }
-
-    private void IterateLevelsRec(Agent agent, List<Agent> agents) {
-      foreach (var child in agent.children) {
-        agents.Add(child);
-      }
-      foreach (var child in agent.children) {
-        IterateLevelsRec(child, agents);
-      }
-    }
-
-    private void IteratePostOrderRec(Agent agent, List<Agent> agents) {
-      foreach (var child in agent.children) {
-        IteratePostOrderRec(child, agents);
-      }
-      agents.Add(agent);
-    }
-
-
-    private static void Swap<T>(ref T a, ref T b) {
-      var temp = a;
-      a = b;
-      b = temp;
-    }
-  }
+  #endregion
 }

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/ContinuedFractionRegression/ContinuedFraction.cs

@@ -1 +1 @@
 using HeuristicLab.Core;
-using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.Algorithms.DataAnalysis.ContinuedFractionRegression {

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/ContinuedFractionRegression/NelderMeadSimplex.cs

@@ -1 +1 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace HeuristicLab.Algorithms.DataAnalysis.ContinuedFractionRegression {

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/ContinuedFractionRegression/Transformation.cs

@@ -1 +1 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using HeuristicLab.Data;
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/ContinuedFractionRegression/Vector.cs

@@ -1 +1 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace HeuristicLab.Algorithms.DataAnalysis.ContinuedFractionRegression {

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -136 +136 @@
     <Compile Include="BaselineClassifiers\OneRClassificationSolution.cs" />
     <Compile Include="BaselineClassifiers\ZeroR.cs" />
+    <Compile Include="ContinuedFractionRegression\Agent.cs" />
     <Compile Include="ContinuedFractionRegression\Algorithm.cs" />
     <Compile Include="ContinuedFractionRegression\ContinuedFraction.cs" />