Changeset 14428

Timestamp:

11/29/16 15:30:27 (6 years ago)

Author:

bburlacu

Message:

#2635: Add analyzer for counting the AdjustedEvaluatedSolutions (according to the actual number of evaluated rows). Add option to preserve compatibility with the standard evaluator. Optimize performance.

Location:

branches/HeuristicLab.OSGAEvaluator

Files:

• HeuristicLab.OSGAEvaluator.sln (modified) (1 diff)
• HeuristicLab.OSGAEvaluator/AdjustedEvaluatedSolutionsAnalyzer.cs (added)
• HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator.csproj (modified) (1 diff)
• HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs (modified) (2 diffs)
• HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs (modified) (10 diffs)

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator.sln

@@ -2 +2 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.25123.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.OSGAEvaluator", "HeuristicLab.OSGAEvaluator\HeuristicLab.OSGAEvaluator.csproj", "{4148A27C-C6F5-44BA-9A57-1460F3758A24}"
 EndProject
 Global
+  GlobalSection(Performance) = preSolution
+    HasPerformanceSessions = true
+  EndGlobalSection
   GlobalSection(SolutionConfigurationPlatforms) = preSolution
     Debug|Any CPU = Debug|Any CPU

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator.csproj

@@ -117 +117 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AdjustedEvaluatedSolutionsAnalyzer.cs" />
     <Compile Include="OSGAPredictionCountsAnalyzer.cs" />
     <Compile Include="Plugin.cs" />

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs

@@ -17 +17 @@
     private const string ResultCollectionParameterName = "Results";
 
-    public ILookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator> EvaluatorParameter {
-      get { return (ILookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator>)Parameters[EvaluatorParameterName]; }
+    public ILookupParameter<SymbolicRegressionSingleObjectiveEvaluator> EvaluatorParameter {
+      get { return (ILookupParameter<SymbolicRegressionSingleObjectiveEvaluator>)Parameters[EvaluatorParameterName]; }
     }
 
     public OSGAPredictionCountsAnalyzer() {
-      Parameters.Add(new LookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator>(EvaluatorParameterName));
+      Parameters.Add(new LookupParameter<SymbolicRegressionSingleObjectiveEvaluator>(EvaluatorParameterName));
     }
 
@@ -33 +33 @@
 
     public override IOperation Apply() {
-      var evaluator = EvaluatorParameter.ActualValue;
+      var evaluator = EvaluatorParameter.ActualValue as SymbolicRegressionSingleObjectiveOsgaEvaluator;
       if (evaluator == null)
         return base.Apply();

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
 using HeuristicLab.Common;
@@ -43 +42 @@
     private const string UseAdaptiveQualityThresholdParameterName = "UseAdaptiveQualityThreshold";
     private const string UseFixedEvaluationIntervalsParameterName = "UseFixedEvaluationIntervals";
+    private const string PreserveResultCompatibilityParameterName = "PreserveEvaluationResultCompatibility";
 
     #region parameters
+    public IFixedValueParameter<BoolValue> PreserveResultCompatibilityParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[PreserveResultCompatibilityParameterName]; }
+    }
     public IFixedValueParameter<BoolValue> UseFixedEvaluationIntervalsParameter {
       get { return (IFixedValueParameter<BoolValue>)Parameters[UseFixedEvaluationIntervalsParameterName]; }
@@ -79 +82 @@
 
     #region parameter properties
+    public bool AggregateStatistics {
+      get { return AggregateStatisticsParameter.Value.Value; }
+      set { AggregateStatisticsParameter.Value.Value = value; }
+    }
+    public bool PreserveResultCompatibility {
+      get { return PreserveResultCompatibilityParameter.Value.Value; }
+      set { PreserveResultCompatibilityParameter.Value.Value = value; }
+    }
     public bool UseFixedEvaluationIntervals {
       get { return UseFixedEvaluationIntervalsParameter.Value.Value; }
@@ -91 +102 @@
       set { RelativeParentChildQualityThresholdParameter.Value.Value = value; }
     }
-
     public double RelativeFitnessEvaluationIntervalSize {
       get { return RelativeFitnessEvaluationIntervalSizeParameter.Value.Value; }
       set { RelativeFitnessEvaluationIntervalSizeParameter.Value.Value = value; }
     }
-
     public IntMatrix RejectedStats {
       get { return RejectedStatsParameter.Value; }
       set { RejectedStatsParameter.Value = value; }
     }
-
     public IntMatrix TotalStats {
       get { return NotRejectedStatsParameter.Value; }
@@ -111 +119 @@
       get { return true; }
     }
+
+    public double AdjustedEvaluatedSolutions { get; set; }
 
     public SymbolicRegressionSingleObjectiveOsgaEvaluator() {
@@ -124 +134 @@
       Parameters.Add(new FixedValueParameter<BoolValue>(UseAdaptiveQualityThresholdParameterName, new BoolValue(false)));
       Parameters.Add(new FixedValueParameter<BoolValue>(UseFixedEvaluationIntervalsParameterName, new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(PreserveResultCompatibilityParameterName, new BoolValue(false)));
     }
 
@@ -136 +147 @@
       if (!Parameters.ContainsKey(UseFixedEvaluationIntervalsParameterName))
         Parameters.Add(new FixedValueParameter<BoolValue>(UseFixedEvaluationIntervalsParameterName, new BoolValue(false)));
+
+      if (!Parameters.ContainsKey(PreserveResultCompatibilityParameterName))
+        Parameters.Add(new FixedValueParameter<BoolValue>(PreserveResultCompatibilityParameterName, new BoolValue(false)));
     }
 
@@ -151 +165 @@
       RejectedStats = new IntMatrix();
       TotalStats = new IntMatrix();
+      AdjustedEvaluatedSolutions = 0;
     }
 
@@ -198 +213 @@
       var estimatedValues = interpreter.GetSymbolicExpressionTreeValues(solution, problemData.Dataset, rows);
       var targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).ToList();
-
       var parentQualities = ParentQualitiesParameter.ActualValue.Select(x => x.Value);
-      var minQuality = parentQualities.Min();
-      var maxQuality = parentQualities.Max();
+      var minQuality = double.MaxValue;
+      var maxQuality = double.MinValue;
+
+      foreach (var quality in parentQualities) {
+        if (minQuality > quality) minQuality = quality;
+        if (maxQuality < quality) maxQuality = quality;
+      }
+
       var comparisonFactor = ComparisonFactorParameter.ActualValue.Value;
       var parentQuality = minQuality + (maxQuality - minQuality) * comparisonFactor;
 
-
-
       #region fixed intervals
       if (UseFixedEvaluationIntervals) {
-        var e = estimatedValues.GetEnumerator();
         double threshold = parentQuality * RelativeParentChildQualityThreshold;
         if (UseAdaptiveQualityThreshold) {
@@ -216 +233 @@
             threshold = parentQuality * (1 - actualSelectionPressure.Value / 100.0);
         }
-
-        var pearsonRCalculator = new OnlinePearsonsRCalculator();
-        var targetValuesEnumerator = targetValues.GetEnumerator();
+        var estimatedEnumerator = estimatedValues.GetEnumerator();
+        var targetEnumerator = targetValues.GetEnumerator();
+
+        var rcalc = new OnlinePearsonsRCalculator();
         var trainingPartitionSize = problemData.TrainingPartition.Size;
         var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
 
-        var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
-        var i = 0;
-        if (aggregateStatistics) {
+        var calculatedRows = 0;
+        #region aggregate statistics
+        if (AggregateStatistics) {
           var trainingEnd = problemData.TrainingPartition.End;
           var qualityPerInterval = new List<double>();
-          while (targetValuesEnumerator.MoveNext() && e.MoveNext()) {
-            pearsonRCalculator.Add(targetValuesEnumerator.Current, e.Current);
-            ++i;
-            if (i % interval == 0 || i == trainingPartitionSize) {
-              var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-              qualityPerInterval.Add(q * q);
-            }
-          }
-          var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+          while (estimatedEnumerator.MoveNext() & targetEnumerator.MoveNext()) {
+            var estimated = estimatedEnumerator.Current;
+            var target = targetEnumerator.Current;
+            rcalc.Add(estimated, target);
+            ++calculatedRows;
+            if (calculatedRows % interval == 0 || calculatedRows == trainingPartitionSize) {
+              var r = rcalc.ErrorState == OnlineCalculatorError.None ? rcalc.R : 0d;
+              qualityPerInterval.Add(r * r);
+            }
+          }
+          double quality;
+          {
+            var r = rcalc.ErrorState != OnlineCalculatorError.None ? 0d : rcalc.R;
+            var actualQuality = r * r;
+            quality = actualQuality;
+            bool predictedRejected = false;
+
+            calculatedRows = 0;
+            foreach (var q in qualityPerInterval) {
+              if (double.IsNaN(q) || !(q > threshold)) {
+                predictedRejected = true;
+                quality = q;
+                break;
+              }
+              ++calculatedRows;
+            }
+
+            var actuallyRejected = !(actualQuality > parentQuality);
+
+            if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
+              RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
+              RejectedStats.RowNames = new[] { "Predicted", "Actual" };
+              RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
+              TotalStats = new IntMatrix(2, 2);
+              TotalStats.RowNames = new[] { "Predicted", "Actual" };
+              TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
+            }
+            // gather some statistics
+            if (predictedRejected) {
+              RejectedStats[0, calculatedRows]++;
+              TotalStats[0, 0]++;
+            } else {
+              TotalStats[0, 1]++;
+            }
+            if (actuallyRejected) {
+              TotalStats[1, 0]++;
+            } else {
+              TotalStats[1, 1]++;
+            }
+            if (predictedRejected && actuallyRejected) {
+              RejectedStats[1, calculatedRows]++;
+            }
+          }
+          return quality;
+        }
+        #endregion
+        else {
+          while (estimatedEnumerator.MoveNext() & targetEnumerator.MoveNext()) {
+            rcalc.Add(targetEnumerator.Current, estimatedEnumerator.Current);
+            ++calculatedRows;
+            if (calculatedRows % interval == 0 || calculatedRows == trainingPartitionSize) {
+              var q = rcalc.ErrorState != OnlineCalculatorError.None ? double.NaN : rcalc.R;
+              var quality = q * q;
+              if (!(quality > threshold)) {
+                AdjustedEvaluatedSolutions += (double)calculatedRows / problemData.TrainingPartition.Size;
+                return quality;
+              }
+            }
+          }
+          var r = rcalc.ErrorState != OnlineCalculatorError.None ? double.NaN : rcalc.R;
           var actualQuality = r * r;
-
-          bool predictedRejected = false;
-
-          i = 0;
-          double quality = actualQuality;
-          foreach (var q in qualityPerInterval) {
-            if (double.IsNaN(q) || !(q > threshold)) {
-              predictedRejected = true;
-              quality = q;
-              break;
-            }
-            ++i;
-          }
-
-          var actuallyRejected = !(actualQuality > parentQuality);
-
-          if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
-            RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
-            RejectedStats.RowNames = new[] { "Predicted", "Actual" };
-            RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
-            TotalStats = new IntMatrix(2, 2);
-            TotalStats.RowNames = new[] { "Predicted", "Actual" };
-            TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
-          }
-          // gather some statistics
-          if (predictedRejected) {
-            RejectedStats[0, i]++;
-            TotalStats[0, 0]++;
-          } else {
-            TotalStats[0, 1]++;
-          }
-          if (actuallyRejected) {
-            TotalStats[1, 0]++;
-          } else {
-            TotalStats[1, 1]++;
-          }
-          if (predictedRejected && actuallyRejected) {
-            RejectedStats[1, i]++;
-          }
-          return quality;
-        } else {
-          while (targetValuesEnumerator.MoveNext() && e.MoveNext()) {
-            pearsonRCalculator.Add(targetValuesEnumerator.Current, e.Current);
-            ++i;
-            if (i % interval == 0 || i == trainingPartitionSize) {
-              var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-              var quality = q * q;
-              if (!(quality > threshold))
-                return quality;
-            }
-          }
-          var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-          var actualQuality = r * r;
+          AdjustedEvaluatedSolutions += 1d;
           return actualQuality;
         }
-      #endregion
+        #endregion
       } else {
         var lsc = new OnlineLinearScalingParameterCalculator();
         var rcalc = new OnlinePearsonsRCalculator();
-        var actualQuality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, SymbolicExpressionTreeParameter.ActualValue, estimationLimits.Lower, estimationLimits.Upper, problemData, rows, true);
-
-        var values = estimatedValues.Zip(targetValues, (es, t) => new { Estimated = es, Target = t });
-        int calculatedRows = 0;
-        double quality = 0.0;
-
-        foreach (var value in values) {
-          lsc.Add(value.Estimated, value.Target);
-          rcalc.Add(value.Estimated, value.Target);
+        var interval = (int)Math.Round(RelativeFitnessEvaluationIntervalSize * problemData.TrainingPartition.Size);
+        var quality = 0d;
+        var calculatedRows = 0;
+
+        var cache = PreserveResultCompatibility ? new List<double>(problemData.TrainingPartition.Size) : null;
+        foreach (var target in estimatedValues.Zip(targetValues, (e, t) => new { EstimatedValue = e, ActualValue = t })) {
+          if (cache != null)
+            cache.Add(target.EstimatedValue);
+
+          lsc.Add(target.EstimatedValue, target.ActualValue);
+          rcalc.Add(target.EstimatedValue, target.ActualValue);
+
           calculatedRows++;
 
-          if (calculatedRows % 5 == 0) {
+          if (calculatedRows % interval != 0) continue;
+
+          var alpha = lsc.Alpha;
+          var beta = lsc.Beta;
+          if (lsc.ErrorState != OnlineCalculatorError.None) {
+            alpha = 0;
+            beta = 1;
+          }
+
+          var calc = (OnlinePearsonsRCalculator)rcalc.Clone();
+          foreach (var t in targetValues.Skip(calculatedRows)) {
+            var s = (t - alpha) / beta; // scaled target
+            calc.Add(s, t); // add pair (scaled, target) to the calculator
+          }
+          var r = calc.ErrorState == OnlineCalculatorError.None ? calc.R : 0d;
+          quality = r * r;
+
+          if (!(quality > parentQuality)) {
+            AdjustedEvaluatedSolutions += (double)calculatedRows / problemData.TrainingPartition.Size;
+            return quality;
+          }
+        }
+        if (PreserveResultCompatibility) {
+          // get quality for all the rows. to ensure reproducibility of results between this evaluator
+          // and the standard one, we calculate the quality in an identical way (otherwise the returned 
+          // quality could be slightly off due to rounding errors (in the range 1e-15 to 1e-16)
+          var applyLinearScaling = ApplyLinearScalingParameter.ActualValue.Value;
+          double r;
+          OnlineCalculatorError calculatorError;
+
+          if (applyLinearScaling) {
             var alpha = lsc.Alpha;
             var beta = lsc.Beta;
-
-            OnlinePearsonsRCalculator calc = (OnlinePearsonsRCalculator)rcalc.Clone();
-            foreach (var t in targetValues.Skip(calculatedRows)) {
-              var scaledTarget = (t - alpha) / beta;
-              calc.Add(scaledTarget, t);
-            }
-
-            var r = calc.ErrorState == OnlineCalculatorError.None ? calc.R : double.NaN;
-            quality = r * r;
-
-            if (quality < parentQuality && actualQuality > parentQuality) {
-              Debugger.Break();
-            }
-            if (quality < parentQuality) return quality;
-          }
+            if (lsc.ErrorState != OnlineCalculatorError.None) {
+              alpha = 0;
+              beta = 1;
+            }
+            var boundedEstimatedValues = cache.Select(x => x * beta + alpha).LimitToRange(estimationLimits.Lower, estimationLimits.Upper);
+            r = OnlinePearsonsRCalculator.Calculate(boundedEstimatedValues, targetValues, out calculatorError);
+          } else {
+            var boundedEstimatedValues = cache.LimitToRange(estimationLimits.Lower, estimationLimits.Upper);
+            r = OnlinePearsonsRCalculator.Calculate(boundedEstimatedValues, targetValues, out calculatorError);
+          }
+          quality = calculatorError == OnlineCalculatorError.None ? r * r : 0d;
         }
-
-        //calculate quality for all rows
-        {
-          var r = rcalc.ErrorState == OnlineCalculatorError.None ? rcalc.R : double.NaN;
-          quality = r * r;
-          if (quality < parentQuality && actualQuality > parentQuality) {
-            Debugger.Break();
-          }
-          if (double.IsNaN(quality)) quality = 0.0;
-          if (quality != actualQuality) Debugger.Break();
-
-          //necessary due to rounding errors and diff in the range of 10E-8
-          quality = actualQuality;
-        }
-
+        AdjustedEvaluatedSolutions += 1d;
         return quality;
       }