Changeset 14231

Timestamp:

08/03/16 18:26:24 (8 years ago)

Author:

bburlacu

Message:

#2635: Add evaluator option to aggregate statistics during evaluation (slower). Improve analyzer.

Location:

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator

Files:

• OSGAPredictionCountsAnalyzer.cs (modified) (4 diffs)
• SymbolicRegressionSingleObjectiveOSGAEvaluator.cs (modified) (4 diffs)

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs

@@ -44 +44 @@
 
       ResultCollection predictionResults;
-      DataTable rejectedStatsTable, totalStatsTable;
+      DataTable rejectedStatsTable, totalStatsTable, rejectedStatsPerGenerationTable, nonRejectedStatsPerGenerationTable;
       if (!ResultCollection.ContainsKey("OS Prediction")) {
         predictionResults = new ResultCollection();
@@ -57 +57 @@
         }
         predictionResults.Add(new Result("Total Stats", totalStatsTable));
-
+        rejectedStatsPerGenerationTable = new DataTable("Rejected Per Generation");
+        nonRejectedStatsPerGenerationTable = new DataTable("Non Rejected Stats per Generation");
+        foreach (var rowName in rejectedStats.RowNames) {
+          rejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
+          rejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
+          nonRejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
+          nonRejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
+        }
+        predictionResults.Add(new Result("Rejected Stats Per Generation", rejectedStatsPerGenerationTable));
+        predictionResults.Add(new Result("Non Rejected Stats Per Generation", nonRejectedStatsPerGenerationTable));
         ResultCollection.Add(new Result("OS Prediction", predictionResults));
       } else {
@@ -63 +72 @@
         rejectedStatsTable = (DataTable)predictionResults["Rejected Stats"].Value;
         totalStatsTable = (DataTable)predictionResults["Total Stats"].Value;
+        rejectedStatsPerGenerationTable = (DataTable)predictionResults["Rejected Stats Per Generation"].Value;
+        nonRejectedStatsPerGenerationTable = (DataTable)predictionResults["Non Rejected Stats Per Generation"].Value;
       }
 
       int i = 0;
       foreach (var rowName in rejectedStats.RowNames) {
-        // add a padding zero below to prevent columns from being cut off to the left
+        // pad values with a 0 on each side to prevent clipping
         rejectedStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, rejectedStats.Columns).Select(j => (double)rejectedStats[i, j])).Concat(new[] { 0d }));
         ++i;
@@ -74 +85 @@
       foreach (var rowName in totalStats.RowNames) {
         totalStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, totalStats.Columns).Select(j => (double)totalStats[i, j])).Concat(new[] { 0d }));
+        var row = rejectedStatsPerGenerationTable.Rows[rowName];
+        var sum = row.Values.Sum();
+        row.Values.Add(totalStats[i, 0] - sum);
+        row = nonRejectedStatsPerGenerationTable.Rows[rowName];
+        sum = row.Values.Sum();
+        row.Values.Add(totalStats[i, 1] - sum);
         ++i;
       }

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

@@ -38 +38 @@
     private const string RelativeFitnessEvaluationIntervalSizeParameterName = "RelativeFitnessEvaluationIntervalSize";
     private const string ResultCollectionParameterName = "Results";
+    private const string AggregateStatisticsParameterName = "AggregateStatistics";
 
     #region parameters
@@ -43 +44 @@
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultCollectionParameterName]; }
     }
-
+    public IValueParameter<BoolValue> AggregateStatisticsParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[AggregateStatisticsParameterName]; }
+    }
     public IValueParameter<IntMatrix> RejectedStatsParameter {
       get { return (IValueParameter<IntMatrix>)Parameters["RejectedStats"]; }
@@ -96 +99 @@
       Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
       Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
+      Parameters.Add(new ValueParameter<BoolValue>(AggregateStatisticsParameterName, new BoolValue(false)));
     }
 
@@ -184 +188 @@
       var targetValuesEnumerator = targetValues.GetEnumerator();
       var estimatedValuesEnumerator = estimatedValues.GetEnumerator();
-
-      var i = 0;
       var trainingPartitionSize = problemData.TrainingPartition.Size;
       var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
-      while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
-        pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
-        ++i;
-        if (i % interval == 0 || i == trainingPartitionSize) {
-          var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-          var quality = q * q;
-          if (!(quality > threshold))
-            return quality;
-        }
+
+      var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
+      var i = 0;
+      if (aggregateStatistics) {
+        var trainingEnd = problemData.TrainingPartition.End;
+        var qualityPerInterval = new List<double>();
+        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
+          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
+          ++i;
+          if (i % interval == 0 || i == trainingPartitionSize) {
+            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+            qualityPerInterval.Add(q * q);
+          }
+        }
+        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+        var actualQuality = r * r;
+
+        bool predictedRejected = false;
+
+        i = 0;
+        double quality = actualQuality;
+        foreach (var q in qualityPerInterval) {
+          if (double.IsNaN(q) || !(q > threshold)) {
+            predictedRejected = true;
+            quality = q;
+            break;
+          }
+          ++i;
+        }
+
+        var actuallyRejected = !(actualQuality > parentQuality);
+
+        if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
+          RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
+          RejectedStats.RowNames = new[] { "Predicted", "Actual" };
+          RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
+          TotalStats = new IntMatrix(2, 2);
+          TotalStats.RowNames = new[] { "Predicted", "Actual" };
+          TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
+        }
+        // gather some statistics
+        if (predictedRejected) {
+          RejectedStats[0, i]++;
+          TotalStats[0, 0]++;
+        } else {
+          TotalStats[0, 1]++;
+        }
+        if (actuallyRejected) {
+          TotalStats[1, 0]++;
+        } else {
+          TotalStats[1, 1]++;
+        }
+        if (predictedRejected && actuallyRejected) {
+          RejectedStats[1, i]++;
+        }
+        return quality;
+      } else {
+        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
+          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
+          ++i;
+          if (i % interval == 0 || i == trainingPartitionSize) {
+            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+            var quality = q * q;
+            if (!(quality > threshold))
+              return quality;
+          }
+        }
+        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
+        var actualQuality = r * r;
+        return actualQuality;
       }
-      var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
-      var actualQuality = r * r;
-      return actualQuality;
     }