Changeset 479 for trunk/sources/HeuristicLab.StructureIdentification/Evaluation

Timestamp:

08/10/08 11:08:07 (16 years ago)

Author:

gkronber

Message:

implemented #242 (All GP evaluators should support the 'UseEstimatedTargetValues' switch for autoregressive modelling).
Also used the chance to remove a lot of the code duplication and thus improve the readability of all GP evaluators.

Location:

trunk/sources/HeuristicLab.StructureIdentification/Evaluation

Files:

• AccuracyEvaluator.cs (modified) (3 diffs)
• ClassificationMeanSquaredErrorEvaluator.cs (modified) (2 diffs)
• CoefficientOfDeterminationEvaluator.cs (modified) (1 diff)
• EarlyStoppingMeanSquaredErrorEvaluator.cs (modified) (2 diffs)
• GPEvaluatorBase.cs (modified) (3 diffs)
• MCCEvaluator.cs (modified) (6 diffs)
• MeanAbsolutePercentageErrorEvaluator.cs (modified) (1 diff)
• MeanSquaredErrorEvaluator.cs (modified) (2 diffs)
• SimpleEvaluator.cs (modified) (2 diffs)
• TheilInequalityCoefficientEvaluator.cs (modified) (3 diffs)
• VarianceAccountedForEvaluator.cs (modified) (1 diff)

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/AccuracyEvaluator.cs

@@ -33 +33 @@
   public class AccuracyEvaluator : GPEvaluatorBase {
     private const double EPSILON = 1.0E-6;
+    private double[] classesArr;
+    private double[] thresholds;
     public override string Description {
       get {
@@ -44 +46 @@
     }
 
-    private double[] original = new double[1];
-    private double[] estimated = new double[1];
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-      int nSamples = trainingEnd-trainingStart;
+    public override IOperation Apply(IScope scope) {
       ItemList<DoubleData> classes = GetVariableValue<ItemList<DoubleData>>("TargetClassValues", scope, true);
-      double[] classesArr = new double[classes.Count];
-      for(int i=0;i<classesArr.Length;i++) classesArr[i] = classes[i].Data;
+      classesArr = new double[classes.Count];
+      for(int i = 0; i < classesArr.Length; i++) classesArr[i] = classes[i].Data;
       Array.Sort(classesArr);
-      double[] thresholds = new double[classes.Count - 1];
-      for(int i=0;i<classesArr.Length-1;i++) {
-        thresholds[i] = (classesArr[i]+classesArr[i+1]) / 2.0;
+      thresholds = new double[classes.Count - 1];
+      for(int i = 0; i < classesArr.Length - 1; i++) {
+        thresholds[i] = (classesArr[i] + classesArr[i + 1]) / 2.0;
       }
 
+      return base.Apply(scope);
+    }
+
+    public override double Evaluate(int start, int end) {
+      int nSamples = end-start;
       int nCorrect = 0;
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double est = evaluator.Evaluate(sample);
-        double origClass = dataset.GetValue(sample, targetVariable);
+      for(int sample = start; sample < end; sample++) {
+        double est = GetEstimatedValue(sample);
+        double origClass = GetOriginalValue(sample);
         double estClass = double.NaN;
         // if estimation is lower than the smallest threshold value -> estimated class is the lower class
@@ -79 +81 @@
         if(Math.Abs(estClass - origClass) < EPSILON) nCorrect++;
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * nSamples;
       return  nCorrect / (double)nSamples;
     }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/ClassificationMeanSquaredErrorEvaluator.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public class ClassificationMeanSquaredErrorEvaluator : GPEvaluatorBase {
-    protected double[] backupValues;
     private const double EPSILON = 1.0E-6;
+    private double[] classesArr;
     public override string Description {
       get {
@@ -46 +46 @@
     }
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
+    public override IOperation Apply(IScope scope) {
       ItemList<DoubleData> classes = GetVariableValue<ItemList<DoubleData>>("TargetClassValues", scope, true);
-      double[] classesArr = new double[classes.Count];
+      classesArr = new double[classes.Count];
       for(int i = 0; i < classesArr.Length; i++) classesArr[i] = classes[i].Data;
       Array.Sort(classesArr);
+      return base.Apply(scope);
+    }
+
+    public override double Evaluate(int start, int end) {
 
       double errorsSquaredSum = 0;
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
-        if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated > targetMean + maximumPunishment) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated < targetMean - maximumPunishment) {
-          estimated = targetMean - maximumPunishment;
-        }
-        double error = estimated - original;
-        // between classes use squared error
-        // on the lower end and upper end only add linear error if the absolute error is larger than 1
-        // the error>1.0 constraint is needed for balance because in the interval ]-1, 1[ the squared error is smaller than the absolute error
-        if(error < -1.0 && IsEqual(original, classesArr[0]) && estimated < classesArr[0] ||
-          error > 1.0 && IsEqual(original, classesArr[classesArr.Length - 1]) && estimated > classesArr[classesArr.Length - 1]) {
-          errorsSquaredSum += Math.Abs(error); // only add linear error below the smallest class or above the largest class
-        } else {
-          errorsSquaredSum += error * error;
+      for(int sample = start; sample < end; sample++) {
+        double estimated = GetEstimatedValue(sample);
+        double original = GetOriginalValue(sample);
+        if(!double.IsNaN(original) && !double.IsInfinity(original)) {
+          double error = estimated - original;
+          // between classes use squared error
+          // on the lower end and upper end only add linear error if the absolute error is larger than 1
+          // the error>1.0 constraint is needed for balance because in the interval ]-1, 1[ the squared error is smaller than the absolute error
+          if(error < -1.0 && IsEqual(original, classesArr[0]) && estimated < classesArr[0] ||
+            error > 1.0 && IsEqual(original, classesArr[classesArr.Length - 1]) && estimated > classesArr[classesArr.Length - 1]) {
+            errorsSquaredSum += Math.Abs(error); // only add linear error below the smallest class or above the largest class
+          } else {
+            errorsSquaredSum += error * error;
+          }
         }
       }
 
-      errorsSquaredSum /= (trainingEnd - trainingStart);
+      errorsSquaredSum /= (end - start);
       if(double.IsNaN(errorsSquaredSum) || double.IsInfinity(errorsSquaredSum)) {
         errorsSquaredSum = double.MaxValue;
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd - trainingStart);
       return errorsSquaredSum;
     }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/CoefficientOfDeterminationEvaluator.cs

@@ -43 +43 @@
     }
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
+    public override double Evaluate(int start, int end) {
       double errorsSquaredSum = 0.0;
       double originalDeviationTotalSumOfSquares = 0.0;
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
+      for(int sample = start; sample < end; sample++) {
+        double estimated = GetEstimatedValue(sample);
+        double original = GetOriginalValue(sample);
         if(!double.IsNaN(original) && !double.IsInfinity(original)) {
-          if(double.IsNaN(estimated) || double.IsInfinity(estimated))
-            estimated = targetMean + maximumPunishment;
-          else if(estimated > (targetMean + maximumPunishment))
-            estimated = targetMean + maximumPunishment;
-          else if(estimated < (targetMean - maximumPunishment))
-            estimated = targetMean - maximumPunishment;
-
           double error = estimated - original;
           errorsSquaredSum += error * error;
 
-          double origDeviation = original - targetMean;
+          double origDeviation = original - TargetMean;
           originalDeviationTotalSumOfSquares += origDeviation * origDeviation;
         }
       }
-
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd - trainingStart);
 
       double quality = 1 - errorsSquaredSum / originalDeviationTotalSumOfSquares;

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/EarlyStoppingMeanSquaredErrorEvaluator.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public class EarlyStoppingMeanSquaredErrorEvaluator : MeanSquaredErrorEvaluator {
+    private double qualityLimit;
     public override string Description {
       get {
@@ -45 +46 @@
     }
 
+    public override IOperation Apply(IScope scope) {
+      qualityLimit = GetVariableValue<DoubleData>("QualityLimit", scope, false).Data;
+      return base.Apply(scope);
+    }
+
     // evaluates the function-tree for the given target-variable and the whole dataset and returns the MSE
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      double qualityLimit = GetVariableValue<DoubleData>("QualityLimit", scope, false).Data;
-      bool useEstimatedValues = GetVariableValue<BoolData>("UseEstimatedTargetValue", scope, false).Data;
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-      int rows = trainingEnd-trainingStart;
-      if(useEstimatedValues && backupValues == null) {
-        backupValues = new double[rows];
-        for(int i = trainingStart; i < trainingEnd; i++) {
-          backupValues[i-trainingStart] = dataset.GetValue(i, targetVariable);
+    public override double Evaluate(int start, int end) {
+      double errorsSquaredSum = 0;
+      int rows = end - start;
+      for(int sample = start; sample < end; sample++) {
+        double estimated = GetEstimatedValue(sample);
+        double original = GetOriginalValue(sample);
+        if(!double.IsNaN(original) && !double.IsInfinity(original)) {
+          double error = estimated - original;
+          errorsSquaredSum += error * error;
+        }
+        // check the limit and stop as soon as we hit the limit
+        if(errorsSquaredSum / rows >= qualityLimit) {
+          return errorsSquaredSum / (sample - start + 1); // return estimated MSE (when the remaining errors are on average the same)
         }
       }
-      double errorsSquaredSum = 0;
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
-        if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated > targetMean + maximumPunishment) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated < targetMean - maximumPunishment) {
-          estimated = targetMean - maximumPunishment;
-        }
-
-        double error = estimated - original;
-        errorsSquaredSum += error * error;
-
-        // check the limit and stop as soon as we hit the limit
-        if(errorsSquaredSum / rows >= qualityLimit) {
-          scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (sample-trainingStart + 1);
-          if(useEstimatedValues) RestoreDataset(dataset, targetVariable, trainingStart, sample);
-          return errorsSquaredSum / (sample-trainingStart + 1); // return estimated MSE (when the remaining errors are on average the same)
-        }
-        if(useEstimatedValues) {
-          dataset.SetValue(sample, targetVariable, estimated);
-        }
-      }
-      if(useEstimatedValues) RestoreDataset(dataset, targetVariable, trainingStart, trainingEnd);
       errorsSquaredSum /= rows;
       if(double.IsNaN(errorsSquaredSum) || double.IsInfinity(errorsSquaredSum)) {
         errorsSquaredSum = double.MaxValue;
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * rows;
       return errorsSquaredSum;
-    }
-
-    private void RestoreDataset(Dataset dataset, int targetVariable, int from, int to) {
-      for(int i = from; i < to; i++) {
-        dataset.SetValue(i, targetVariable, backupValues[i-from]);
-      }
     }
   }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/GPEvaluatorBase.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public abstract class GPEvaluatorBase : OperatorBase {
-    protected double maximumPunishment;
-    protected int treeSize;
-    protected double totalEvaluatedNodes;
-    protected IEvaluator evaluator;
+    private IEvaluator evaluator;
+    private int targetVariable;
+    private int trainingStart;
+    private int trainingEnd;
+    private bool useEstimatedValues;
+    private double[] backupValues;
+    private int evaluatedSamples;
+    private double estimatedValueMax;
+    private double estimatedValueMin;
+    private int treeSize;
+    private double totalEvaluatedNodes;
+    private Dataset dataset;
+    private double targetMean;
+    protected double TargetMean { get { return targetMean; } }
 
     public GPEvaluatorBase()
@@ -47 +57 @@
       AddVariableInfo(new VariableInfo("TrainingSamplesStart", "Start index of training samples in dataset", typeof(IntData), VariableKind.In));
       AddVariableInfo(new VariableInfo("TrainingSamplesEnd", "End index of training samples in dataset", typeof(IntData), VariableKind.In));
+      AddVariableInfo(new VariableInfo("UseEstimatedTargetValue", "Wether to use the original (measured) or the estimated (calculated) value for the targat variable when doing autoregressive modelling", typeof(BoolData), VariableKind.In));
       AddVariableInfo(new VariableInfo("Quality", "The evaluated quality of the model", typeof(DoubleData), VariableKind.New));
     }
 
     public override IOperation Apply(IScope scope) {
-      int targetVariable = GetVariableValue<IntData>("TargetVariable", scope, true).Data;
-      Dataset dataset = GetVariableValue<Dataset>("Dataset", scope, true);
+      // get all variable values
+      targetVariable = GetVariableValue<IntData>("TargetVariable", scope, true).Data;
+      dataset = GetVariableValue<Dataset>("Dataset", scope, true);
       IFunctionTree functionTree = GetVariableValue<IFunctionTree>("FunctionTree", scope, true);
-      this.maximumPunishment = GetVariableValue<DoubleData>("PunishmentFactor", scope, true).Data * dataset.GetRange(targetVariable);
-      this.treeSize = scope.GetVariableValue<IntData>("TreeSize", false).Data;
-      this.totalEvaluatedNodes = scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data;
+      double maximumPunishment = GetVariableValue<DoubleData>("PunishmentFactor", scope, true).Data * dataset.GetRange(targetVariable);
+      treeSize = scope.GetVariableValue<IntData>("TreeSize", false).Data;
+      totalEvaluatedNodes = scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data;
+      trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
+      trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
+      useEstimatedValues = GetVariableValue<BoolData>("UseEstimatedTargetValue", scope, true).Data;
+      // prepare for autoregressive modelling by saving the original values of the target-variable to a backup array
+      if(useEstimatedValues && backupValues == null) {
+        backupValues = new double[trainingEnd - trainingStart];
+        for(int i = trainingStart; i < trainingEnd; i++) {
+          backupValues[i - trainingStart] = dataset.GetValue(i, targetVariable);
+        }
+      }
+      // get the mean of the values of the target variable to determin the max and min bounds of the estimated value
+      targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
+      estimatedValueMin = targetMean - maximumPunishment;
+      estimatedValueMax = targetMean + maximumPunishment;
+
+      // initialize and reset the evaluator
       if(evaluator == null) evaluator = functionTree.CreateEvaluator(dataset);
       evaluator.ResetEvaluator(functionTree);
-      double result = Evaluate(scope, functionTree, targetVariable, dataset);
+      evaluatedSamples = 0;
 
+      // calculate the quality measure
+      double result = Evaluate(trainingStart, trainingEnd);
+
+      // restore the values of the target variable from the backup array if necessary
+      if(useEstimatedValues) RestoreDataset(dataset, targetVariable, trainingStart, trainingEnd);
+      // update the value of total evaluated nodes
+      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * evaluatedSamples;
+      // write the calculate quality value
       DoubleData quality = GetVariableValue<DoubleData>("Quality", scope, false, false);
       if(quality == null) {
@@ -67 +103 @@
         quality.Data = result;
       }
-
       return null;
     }
 
-    public abstract double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset);
+    private void RestoreDataset(Dataset dataset, int targetVariable, int from, int to) {
+      for(int i = from; i < to; i++) {
+        dataset.SetValue(i, targetVariable, backupValues[i - from]);
+      }
+    }
+
+    public abstract double Evaluate(int start, int end);
+
+    public double GetOriginalValue(int sample) {
+      if(useEstimatedValues)
+        return backupValues[sample - trainingStart];
+      else
+        return dataset.GetValue(sample, targetVariable);
+    }
+
+    public double GetEstimatedValue(int sample) {
+      evaluatedSamples++;
+      double estimated = evaluator.Evaluate(sample);
+      if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {
+        estimated = estimatedValueMax;
+      } else if(estimated > estimatedValueMax) {
+        estimated = estimatedValueMax;
+      } else if(estimated < estimatedValueMin) {
+        estimated = estimatedValueMin;
+      }
+
+      if(useEstimatedValues) {
+        dataset.SetValue(sample, targetVariable, estimated);
+      }
+
+      return estimated;
+    }
   }
 }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MCCEvaluator.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public class MCCEvaluator : GPEvaluatorBase {
+    private double limit;
+    private double[] original = new double[1];
+    private double[] estimated = new double[1];
     public override string Description {
       get {
@@ -37 +40 @@
       }
     }
-
     public MCCEvaluator()
       : base() {
@@ -43 +45 @@
     }
 
-    private double[] original = new double[1];
-    private double[] estimated = new double[1];
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-      int nSamples = trainingEnd-trainingStart;
-      double limit = GetVariableValue<DoubleData>("ClassSeparation", scope, true).Data;
+    public override IOperation Apply(IScope scope) {
+      limit = GetVariableValue<DoubleData>("ClassSeparation", scope, true).Data;
+      return base.Apply(scope);
+    }
+
+    public override double Evaluate(int start, int end) {
+      int nSamples = end - start;
       if(estimated.Length != nSamples) {
         estimated = new double[nSamples];
@@ -57 +59 @@
       double positive = 0;
       double negative = 0;
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double est = evaluator.Evaluate(sample);
-        double orig = dataset.GetValue(sample, targetVariable);
-        if(double.IsNaN(est) || double.IsInfinity(est)) {
-          est = targetMean + maximumPunishment;
-        } else if(est > targetMean + maximumPunishment) {
-          est = targetMean + maximumPunishment;
-        } else if(est < targetMean - maximumPunishment) {
-          est = targetMean - maximumPunishment;
-        }
-        estimated[sample-trainingStart] = est;
-        original[sample-trainingStart] = orig;
+      for(int sample = start; sample < end; sample++) {
+        double est = GetEstimatedValue(sample);
+        double orig = GetOriginalValue(sample);
+        estimated[sample - start] = est;
+        original[sample - start] = orig;
         if(orig >= limit) positive++;
         else negative++;
@@ -79 +73 @@
       double tn = negative;
       double fp = 0;
-      for(int i = original.Length-1; i >= 0 ; i--) {
+      for(int i = original.Length - 1; i >= 0; i--) {
         if(original[i] >= limit) {
           tp++; fn--;
@@ -90 +84 @@
         }
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd - trainingStart);
       return best_mcc;
     }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MeanAbsolutePercentageErrorEvaluator.cs

@@ -43 +43 @@
     }
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
+    public override double Evaluate(int start, int end) {
       double errorsSum = 0.0;
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
+      for(int sample = start; sample < end; sample++) {
+        double estimated = GetEstimatedValue(sample);
+        double original = GetOriginalValue(sample);
         if(!double.IsNaN(original) && !double.IsInfinity(original)) {
-          if(double.IsNaN(estimated) || double.IsInfinity(estimated))
-            estimated = maximumPunishment;
-          else if(estimated > maximumPunishment)
-            estimated = maximumPunishment;
-          else if(estimated < -maximumPunishment)
-            estimated = -maximumPunishment;
-
           double percent_error = Math.Abs((estimated - original) / original);
           errorsSum += percent_error;
         }
       }
-      int nSamples = trainingEnd - trainingStart;
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * nSamples;
-      double quality = errorsSum / nSamples;
+      double quality = errorsSum / (end - start);
       if(double.IsNaN(quality) || double.IsInfinity(quality))
         quality = double.MaxValue;

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MeanSquaredErrorEvaluator.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public class MeanSquaredErrorEvaluator : GPEvaluatorBase {
-    protected double[] backupValues;
     public override string Description {
       get {
@@ -42 +41 @@
     public MeanSquaredErrorEvaluator()
       : base() {
-      AddVariableInfo(new VariableInfo("UseEstimatedTargetValue", "Wether to use the original (measured) or the estimated (calculated) value for the targat variable when doing autoregressive modelling", typeof(BoolData), VariableKind.In));
-      GetVariableInfo("UseEstimatedTargetValue").Local = true;
-      AddVariable(new HeuristicLab.Core.Variable("UseEstimatedTargetValue", new BoolData(false)));
     }
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
+    public override double Evaluate(int start, int end) {
       double errorsSquaredSum = 0;
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      bool useEstimatedValues = GetVariableValue<BoolData>("UseEstimatedTargetValue", scope, false).Data;
-      if(useEstimatedValues && backupValues == null) {
-        backupValues = new double[trainingEnd - trainingStart];
-        for(int i = trainingStart; i < trainingEnd; i++) {
-          backupValues[i-trainingStart] = dataset.GetValue(i, targetVariable);
-        }
-      }
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
-        if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated > targetMean + maximumPunishment) {
-          estimated = targetMean + maximumPunishment;
-        } else if(estimated < targetMean - maximumPunishment) {
-          estimated = targetMean - maximumPunishment;
-        }
-        double error = estimated - original;
-        errorsSquaredSum += error * error;
-        if(useEstimatedValues) {
-          dataset.SetValue(sample, targetVariable, estimated);
+      for(int sample = start; sample < end; sample++) {
+        double original = GetOriginalValue(sample);
+        double estimated = GetEstimatedValue(sample);
+        if(!double.IsNaN(original) && !double.IsInfinity(original)) {
+          double error = estimated - original;
+          errorsSquaredSum += error * error;
         }
       }
 
-      if(useEstimatedValues) RestoreDataset(dataset, targetVariable, trainingStart, trainingEnd);
-      errorsSquaredSum /= (trainingEnd-trainingStart);
+      errorsSquaredSum /= (end - start);
       if(double.IsNaN(errorsSquaredSum) || double.IsInfinity(errorsSquaredSum)) {
         errorsSquaredSum = double.MaxValue;
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd-trainingStart);
       return errorsSquaredSum;
-    }
-
-    private void RestoreDataset(Dataset dataset, int targetVariable, int from, int to) {
-      for(int i = from; i < to; i++) {
-        dataset.SetValue(i, targetVariable, backupValues[i-from]);
-      }
     }
   }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/SimpleEvaluator.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.StructureIdentification {
-  public class SimpleEvaluator : OperatorBase {
-    protected int treeSize;
-    protected double totalEvaluatedNodes;
-    private IEvaluator evaluator;
-
+  public class SimpleEvaluator : GPEvaluatorBase {
+    private ItemList values;
     public SimpleEvaluator()
       : base() {
-      AddVariableInfo(new VariableInfo("FunctionTree", "The function tree that should be evaluated", typeof(IFunctionTree), VariableKind.In));
-      AddVariableInfo(new VariableInfo("TreeSize", "Size (number of nodes) of the tree to evaluate", typeof(IntData), VariableKind.In));
-      AddVariableInfo(new VariableInfo("Dataset", "Dataset with all samples on which to apply the function", typeof(Dataset), VariableKind.In));
-      AddVariableInfo(new VariableInfo("TargetVariable", "Index of the column of the dataset that holds the target variable", typeof(IntData), VariableKind.In));
-      AddVariableInfo(new VariableInfo("TotalEvaluatedNodes", "Number of evaluated nodes", typeof(DoubleData), VariableKind.In | VariableKind.Out));
-      AddVariableInfo(new VariableInfo("TrainingSamplesStart", "Start index of training samples in dataset", typeof(IntData), VariableKind.In));
-      AddVariableInfo(new VariableInfo("TrainingSamplesEnd", "End index of training samples in dataset", typeof(IntData), VariableKind.In));
       AddVariableInfo(new VariableInfo("Values", "The values of the target variable as predicted by the model and the original value of the target variable", typeof(ItemList), VariableKind.New | VariableKind.Out));
     }
 
     public override IOperation Apply(IScope scope) {
-      int targetVariable = GetVariableValue<IntData>("TargetVariable", scope, true).Data;
-      Dataset dataset = GetVariableValue<Dataset>("Dataset", scope, true);
-      IFunctionTree functionTree = GetVariableValue<IFunctionTree>("FunctionTree", scope, true);
-      this.treeSize = scope.GetVariableValue<IntData>("TreeSize", false).Data;
-      this.totalEvaluatedNodes = scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data;
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-
-      ItemList values = GetVariableValue<ItemList>("Values", scope, false, false);
+      values = GetVariableValue<ItemList>("Values", scope, false, false);
       if(values == null) {
         values = new ItemList();
@@ -67 +49 @@
       }
       values.Clear();
-      if(evaluator == null) evaluator = functionTree.CreateEvaluator(dataset);
-      evaluator.ResetEvaluator(functionTree);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
+      return base.Apply(scope);
+    }
+
+    public override double Evaluate(int start, int end) {
+      for(int sample = start; sample < end; sample++) {
         ItemList row = new ItemList();
-        row.Add(new DoubleData(evaluator.Evaluate(sample)));
-        row.Add(new DoubleData(dataset.GetValue(sample, targetVariable)));
+        row.Add(new DoubleData(GetEstimatedValue(sample)));
+        row.Add(new DoubleData(GetOriginalValue(sample)));
         values.Add(row);
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd - trainingStart);
-      return null;
+      return double.NaN;
     }
   }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/TheilInequalityCoefficientEvaluator.cs

@@ -32 +32 @@
 namespace HeuristicLab.StructureIdentification {
   public class TheilInequalityCoefficientEvaluator : GPEvaluatorBase {
+    private bool differential;
     public override string Description {
       get {
@@ -44 +45 @@
     }
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-      bool difference = GetVariableValue<BoolData>("Differential", scope, true).Data;
+    public override IOperation Apply(IScope scope) {
+      differential = GetVariableValue<BoolData>("Differential", scope, true).Data;
+      return base.Apply(scope);
+    }
+
+    public override double Evaluate(int start, int end) {
       double errorsSquaredSum = 0.0;
       double estimatedSquaredSum = 0.0;
       double originalSquaredSum = 0.0;
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
+      for(int sample = start; sample < end; sample++) {
         double prevValue = 0.0;
-        if(difference) prevValue = dataset.GetValue(sample - 1, targetVariable);
-        double estimatedChange = evaluator.Evaluate(sample) - prevValue;
-        double originalChange = dataset.GetValue(sample, targetVariable) - prevValue;
+        if(differential) prevValue = GetOriginalValue(sample - 1);
+        double estimatedChange = GetEstimatedValue(sample) - prevValue;
+        double originalChange = GetOriginalValue(sample) - prevValue;
         if(!double.IsNaN(originalChange) && !double.IsInfinity(originalChange)) {
-          if(double.IsNaN(estimatedChange) || double.IsInfinity(estimatedChange))
-            estimatedChange = maximumPunishment;
-          else if(estimatedChange > maximumPunishment)
-            estimatedChange = maximumPunishment;
-          else if(estimatedChange < -maximumPunishment)
-            estimatedChange = - maximumPunishment;
-
           double error = estimatedChange - originalChange;
           errorsSquaredSum += error * error;
@@ -70 +66 @@
         }
       }
-      int nSamples = trainingEnd - trainingStart;
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * nSamples;
+      int nSamples = end - start;
       double quality = Math.Sqrt(errorsSquaredSum / nSamples) / (Math.Sqrt(estimatedSquaredSum / nSamples) + Math.Sqrt(originalSquaredSum / nSamples));
-      if(double.IsNaN(quality) || double.IsInfinity(quality)) 
+      if(double.IsNaN(quality) || double.IsInfinity(quality))
         quality = double.MaxValue;
       return quality;

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/VarianceAccountedForEvaluator.cs

@@ -53 +53 @@
 
 
-    public override double Evaluate(IScope scope, IFunctionTree functionTree, int targetVariable, Dataset dataset) {
-      int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
-      int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
-      double[] errors = new double[trainingEnd-trainingStart];
-      double[] originalTargetVariableValues = new double[trainingEnd-trainingStart];
-      double targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
-      for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = evaluator.Evaluate(sample);
-        double original = dataset.GetValue(sample, targetVariable);
+    public override double Evaluate(int start, int end) {
+      int nSamples = end - start;
+      double[] errors = new double[nSamples];
+      double[] originalTargetVariableValues = new double[nSamples];
+      for(int sample = start; sample < end; sample++) {
+        double estimated = GetEstimatedValue(sample);
+        double original = GetOriginalValue(sample);
         if(!double.IsNaN(original) && !double.IsInfinity(original)) {
-          if(double.IsNaN(estimated) || double.IsInfinity(estimated))
-            estimated = targetMean + maximumPunishment;
-          else if(estimated > (targetMean + maximumPunishment))
-            estimated = targetMean + maximumPunishment;
-          else if(estimated < (targetMean - maximumPunishment))
-            estimated = targetMean - maximumPunishment;
+          errors[sample - start] = original - estimated;
+          originalTargetVariableValues[sample - start] = original;
         }
-
-        errors[sample-trainingStart] = original - estimated;
-        originalTargetVariableValues[sample-trainingStart] = original;
       }
-      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * (trainingEnd-trainingStart);
-
       double errorsVariance = Statistics.Variance(errors);
       double originalsVariance = Statistics.Variance(originalTargetVariableValues);