Changeset 14279 for branches/HeuristicLab.OSGAEvaluator

Timestamp:

09/13/16 16:43:31 (8 years ago)

Author:

bburlacu

Message:

#2635: Implement an alternative way of assessing which offspring should be rejected early: the child is evaluated on part of the training data and the remaining rows are assumed to be perfectly correlated with the target. If the offspring selection criteria is not fulfilled, we can reject early. Otherwise the child is evaluated on an additional slice of the training and so on.

File:

• branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs (modified) (6 diffs)

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

@@ -39 +39 @@
     private const string ResultCollectionParameterName = "Results";
     private const string AggregateStatisticsParameterName = "AggregateStatistics";
+    private const string ActualSelectionPressureParameterName = "SelectionPressure";
+    private const string UseAdaptiveQualityThresholdParameterName = "UseAdaptiveQualityThreshold";
+    private const string UseFixedEvaluationIntervalsParameterName = "UseFixedEvaluationIntervals";
 
     #region parameters
+    public IFixedValueParameter<BoolValue> UseFixedEvaluationIntervalsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseFixedEvaluationIntervalsParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> UseAdaptiveQualityThresholdParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseAdaptiveQualityThresholdParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> ActualSelectionPressureParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[ActualSelectionPressureParameterName]; }
+    }
     public ILookupParameter<ResultCollection> ResultCollectionParameter {
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultCollectionParameterName]; }
@@ -66 +78 @@
 
     #region parameter properties
+    public bool UseFixedEvaluationIntervals {
+      get { return UseFixedEvaluationIntervalsParameter.Value.Value; }
+      set { UseFixedEvaluationIntervalsParameter.Value.Value = value; }
+    }
+    public bool UseAdaptiveQualityThreshold {
+      get { return UseAdaptiveQualityThresholdParameter.Value.Value; }
+      set { UseAdaptiveQualityThresholdParameter.Value.Value = value; }
+    }
     public double RelativeParentChildQualityThreshold {
       get { return RelativeParentChildQualityThresholdParameter.Value.Value; }
@@ -100 +120 @@
       Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
       Parameters.Add(new ValueParameter<BoolValue>(AggregateStatisticsParameterName, new BoolValue(false)));
+      Parameters.Add(new LookupParameter<DoubleValue>(ActualSelectionPressureParameterName));
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseAdaptiveQualityThresholdParameterName, new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseFixedEvaluationIntervalsParameterName, new BoolValue(false)));
     }
 
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
-      if (!Parameters.ContainsKey(ResultCollectionParameterName))
-        Parameters.Add(new LookupParameter<ResultCollection>(ResultCollectionParameterName));
-
-      if (!Parameters.ContainsKey("ParentQualities"))
-        Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ParentQualities") { ActualName = "Quality" });
-
-      if (!Parameters.ContainsKey("RejectedStats"))
-        Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
-
-      if (!Parameters.ContainsKey("TotalStats"))
-        Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
+      if (!Parameters.ContainsKey(ActualSelectionPressureParameterName))
+        Parameters.Add(new LookupParameter<DoubleValue>(ActualSelectionPressureParameterName));
+
+      if (!Parameters.ContainsKey(UseAdaptiveQualityThresholdParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(UseAdaptiveQualityThresholdParameterName, new BoolValue(false)));
+
+      if (!Parameters.ContainsKey(UseFixedEvaluationIntervalsParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(UseFixedEvaluationIntervalsParameterName, new BoolValue(false)));
     }
 
@@ -146 +166 @@
       // parent subscopes are not present during evaluation of the initial population
       if (parentQualities.Length > 0) {
-        quality = Calculate(interpreter, solution, estimationLimits, problemData, rows, applyLinearScaling);
+        quality = Calculate(interpreter, solution, estimationLimits, problemData, rows);
       } else {
         quality = Calculate(interpreter, solution, estimationLimits.Lower, estimationLimits.Upper, problemData, rows, applyLinearScaling);
@@ -174 +194 @@
     }
 
-    private double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree solution, DoubleLimit estimationLimits, IRegressionProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling) {
+    private double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree solution, DoubleLimit estimationLimits, IRegressionProblemData problemData, IEnumerable<int> rows) {
       var estimatedValues = interpreter.GetSymbolicExpressionTreeValues(solution, problemData.Dataset, rows).LimitToRange(estimationLimits.Lower, estimationLimits.Upper);
-      var targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      var targetValues = problemData.Dataset.GetReadOnlyDoubleValues(problemData.TargetVariable);
 
       var parentQualities = ParentQualitiesParameter.ActualValue.Select(x => x.Value);
@@ -183 +203 @@
       var comparisonFactor = ComparisonFactorParameter.ActualValue.Value;
       var parentQuality = minQuality + (maxQuality - minQuality) * comparisonFactor;
-      var threshold = parentQuality * RelativeParentChildQualityThreshold;
-
-      var pearsonRCalculator = new OnlinePearsonsRCalculator();
-      var targetValuesEnumerator = targetValues.GetEnumerator();
-      var estimatedValuesEnumerator = estimatedValues.GetEnumerator();
-      var trainingPartitionSize = problemData.TrainingPartition.Size;
-      var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
-
-      var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
-      var i = 0;
-      if (aggregateStatistics) {
-        var trainingEnd = problemData.TrainingPartition.End;
-        var qualityPerInterval = new List<double>();
-        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
-          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
-          ++i;
-          if (i % interval == 0 || i == trainingPartitionSize) {
-            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-            qualityPerInterval.Add(q * q);
-          }
+
+      var e = estimatedValues.GetEnumerator();
+
+      if (UseFixedEvaluationIntervals) {
+        double threshold = parentQuality * RelativeParentChildQualityThreshold;
+        if (UseAdaptiveQualityThreshold) {
+          var actualSelectionPressure = ActualSelectionPressureParameter.ActualValue;
+          if (actualSelectionPressure != null)
+            threshold = parentQuality * (1 - actualSelectionPressure.Value / 100.0);
         }
-        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-        var actualQuality = r * r;
-
-        bool predictedRejected = false;
-
-        i = 0;
-        double quality = actualQuality;
-        foreach (var q in qualityPerInterval) {
-          if (double.IsNaN(q) || !(q > threshold)) {
-            predictedRejected = true;
-            quality = q;
+
+        var pearsonRCalculator = new OnlinePearsonsRCalculator();
+        var targetValuesEnumerator = targetValues.GetEnumerator();
+        var trainingPartitionSize = problemData.TrainingPartition.Size;
+        var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
+
+        var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
+        var i = 0;
+        if (aggregateStatistics) {
+          var trainingEnd = problemData.TrainingPartition.End;
+          var qualityPerInterval = new List<double>();
+          while (targetValuesEnumerator.MoveNext() && e.MoveNext()) {
+            pearsonRCalculator.Add(targetValuesEnumerator.Current, e.Current);
+            ++i;
+            if (i % interval == 0 || i == trainingPartitionSize) {
+              var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+              qualityPerInterval.Add(q * q);
+            }
+          }
+          var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+          var actualQuality = r * r;
+
+          bool predictedRejected = false;
+
+          i = 0;
+          double quality = actualQuality;
+          foreach (var q in qualityPerInterval) {
+            if (double.IsNaN(q) || !(q > threshold)) {
+              predictedRejected = true;
+              quality = q;
+              break;
+            }
+            ++i;
+          }
+
+          var actuallyRejected = !(actualQuality > parentQuality);
+
+          if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
+            RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
+            RejectedStats.RowNames = new[] { "Predicted", "Actual" };
+            RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
+            TotalStats = new IntMatrix(2, 2);
+            TotalStats.RowNames = new[] { "Predicted", "Actual" };
+            TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
+          }
+          // gather some statistics
+          if (predictedRejected) {
+            RejectedStats[0, i]++;
+            TotalStats[0, 0]++;
+          } else {
+            TotalStats[0, 1]++;
+          }
+          if (actuallyRejected) {
+            TotalStats[1, 0]++;
+          } else {
+            TotalStats[1, 1]++;
+          }
+          if (predictedRejected && actuallyRejected) {
+            RejectedStats[1, i]++;
+          }
+          return quality;
+        } else {
+          while (targetValuesEnumerator.MoveNext() && e.MoveNext()) {
+            pearsonRCalculator.Add(targetValuesEnumerator.Current, e.Current);
+            ++i;
+            if (i % interval == 0 || i == trainingPartitionSize) {
+              var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+              var quality = q * q;
+              if (!(quality > threshold))
+                return quality;
+            }
+          }
+          var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+          var actualQuality = r * r;
+          return actualQuality;
+        }
+      } else {
+        var calculator = new OnlinePearsonsRCalculator();
+        var trainingPartitionSize = problemData.TrainingPartition.Size;
+        var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
+        double quality = double.NaN;
+        var estimated = new List<double>(); // save estimated values in a list so we don't re-evaluate 
+        // use the actual estimated values for the first i * interval rows of the training partition and and assume the remaining rows are perfectly correlated
+        // if the quality of the individual still falls below the parent quality, then we can reject it sooner, otherwise as i increases the whole estimated series will be used
+        for (int i = 0; i < trainingPartitionSize; i += interval) {
+          int j = i;
+          int end = Math.Min(trainingPartitionSize, i + interval);
+          while (j < end && e.MoveNext()) {
+            estimated.Add(e.Current);
+            j++;
+          }
+
+          var start = problemData.TrainingPartition.Start;
+          calculator.Reset();
+          // add (estimated, target) pairs to the calculator
+          for (j = 0; j < end; ++j)
+            calculator.Add(estimated[j], targetValues[j + start]);
+          // add (target, target) pairs to the calculator (simulate perfect correlation on the remaining rows)
+          for (; j < trainingPartitionSize; ++j) {
+            var index = j + start;
+            calculator.Add(targetValues[index], targetValues[index]);
+          }
+          var r = calculator.ErrorState == OnlineCalculatorError.None ? calculator.R : double.NaN;
+          quality = r * r;
+          if (!(quality > parentQuality))
             break;
-          }
-          ++i;
-        }
-
-        var actuallyRejected = !(actualQuality > parentQuality);
-
-        if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
-          RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
-          RejectedStats.RowNames = new[] { "Predicted", "Actual" };
-          RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
-          TotalStats = new IntMatrix(2, 2);
-          TotalStats.RowNames = new[] { "Predicted", "Actual" };
-          TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
-        }
-        // gather some statistics
-        if (predictedRejected) {
-          RejectedStats[0, i]++;
-          TotalStats[0, 0]++;
-        } else {
-          TotalStats[0, 1]++;
-        }
-        if (actuallyRejected) {
-          TotalStats[1, 0]++;
-        } else {
-          TotalStats[1, 1]++;
-        }
-        if (predictedRejected && actuallyRejected) {
-          RejectedStats[1, i]++;
         }
         return quality;
-      } else {
-        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
-          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
-          ++i;
-          if (i % interval == 0 || i == trainingPartitionSize) {
-            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-            var quality = q * q;
-            if (!(quality > threshold))
-              return quality;
-          }
-        }
-        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-        var actualQuality = r * r;
-        return actualQuality;
       }
     }