Changeset 14104

Timestamp:

07/18/16 17:14:33 (8 years ago)

Author:

bburlacu

Message:

#2635: Improve performance and accuracy of evaluator and analyzer.

Location:

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator

Files:

• OSGAPredictionCountsAnalyzer.cs (modified) (3 diffs)
• SymbolicRegressionSingleObjectiveOSGAEvaluator.cs (modified) (9 diffs)

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs

@@ -1 @@
-using HeuristicLab.Common;
+using System.Linq;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
 
 namespace HeuristicLab.OSGAEvaluator {
@@ -9 +14 @@
 
   public class OSGAPredictionCountsAnalyzer : SymbolicDataAnalysisSingleObjectiveAnalyzer {
+    private const string EvaluatorParameterName = "Evaluator";
+    private const string ResultCollectionParameterName = "Results";
+
+    public ILookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator> EvaluatorParameter {
+      get { return (ILookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator>)Parameters[EvaluatorParameterName]; }
+    }
+
+    public OSGAPredictionCountsAnalyzer() {
+      Parameters.Add(new LookupParameter<SymbolicRegressionSingleObjectiveOsgaEvaluator>(EvaluatorParameterName));
+    }
 
     protected OSGAPredictionCountsAnalyzer(OSGAPredictionCountsAnalyzer original, Cloner cloner) : base(original, cloner) {
@@ -16 +31 @@
       return new OSGAPredictionCountsAnalyzer(this, cloner);
     }
+
+    public override IOperation Apply() {
+      var evaluator = EvaluatorParameter.ActualValue;
+      if (evaluator == null)
+        return base.Apply();
+
+      var rejectedStats = evaluator.RejectedStats;
+      var totalStats = evaluator.TotalStats;
+
+      if (rejectedStats.Rows == 0 || totalStats.Rows == 0)
+        return base.Apply();
+
+      ResultCollection predictionResults;
+      DataTable rejectedStatsTable, totalStatsTable;
+      if (!ResultCollection.ContainsKey("OS Prediction")) {
+        predictionResults = new ResultCollection();
+        rejectedStatsTable = new DataTable("Rejected Stats");
+        foreach (var rowName in rejectedStats.RowNames) {
+          rejectedStatsTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Columns, LineWidth = 1 } });
+        }
+        predictionResults.Add(new Result("Rejected Stats", rejectedStatsTable));
+        totalStatsTable = new DataTable("Total Stats");
+        foreach (var rowName in totalStats.RowNames) {
+          totalStatsTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Columns, LineWidth = 1 } });
+        }
+        predictionResults.Add(new Result("Total Stats", totalStatsTable));
+
+        ResultCollection.Add(new Result("OS Prediction", predictionResults));
+      } else {
+        predictionResults = (ResultCollection)ResultCollection["OS Prediction"].Value;
+        rejectedStatsTable = (DataTable)predictionResults["Rejected Stats"].Value;
+        totalStatsTable = (DataTable)predictionResults["Total Stats"].Value;
+      }
+
+      int i = 0;
+      foreach (var rowName in rejectedStats.RowNames) {
+        // add a padding zero below to prevent columns from being cut off to the left
+        rejectedStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, rejectedStats.Columns).Select(j => (double)rejectedStats[i, j])).Concat(new[] { 0d }));
+        ++i;
+      }
+      i = 0;
+      foreach (var rowName in totalStats.RowNames) {
+        totalStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, totalStats.Columns).Select(j => (double)totalStats[i, j])).Concat(new[] { 0d }));
+        ++i;
+      }
+      return base.Apply();
+    }
   }
 }

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
-using HeuristicLab.Analysis;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -44 +43 @@
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultCollectionParameterName]; }
     }
-    public IFixedValueParameter<IntValue> CorrectlyRejectedParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["CorrectlyRejected"]; }
-    }
-    public IFixedValueParameter<IntValue> IncorrectlyRejectedParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["IncorrectlyRejected"]; }
-    }
-    public IFixedValueParameter<IntValue> CorrectlyNotRejectedParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["CorrectlyNotRejected"]; }
-    }
-    public IFixedValueParameter<IntValue> IncorrectlyNotRejectedParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["IncorrectlyNotRejected"]; }
+
+    public IValueParameter<IntMatrix> RejectedStatsParameter {
+      get { return (IValueParameter<IntMatrix>)Parameters["RejectedStats"]; }
+    }
+    public IValueParameter<IntMatrix> NotRejectedStatsParameter {
+      get { return (IValueParameter<IntMatrix>)Parameters["TotalStats"]; }
     }
     public IValueLookupParameter<DoubleValue> ComparisonFactorParameter {
@@ -79 +73 @@
     }
 
-    public int CorrectlyRejected {
-      get { return CorrectlyRejectedParameter.Value.Value; }
-      set { CorrectlyRejectedParameter.Value.Value = value; }
-    }
-
-    public int CorrectlyNotRejected {
-      get { return CorrectlyNotRejectedParameter.Value.Value; }
-      set { CorrectlyNotRejectedParameter.Value.Value = value; }
-    }
-
-    public int IncorrectlyRejected {
-      get { return IncorrectlyRejectedParameter.Value.Value; }
-      set { IncorrectlyRejectedParameter.Value.Value = value; }
-    }
-
-    public int IncorrectlyNotRejected {
-      get { return IncorrectlyNotRejectedParameter.Value.Value; }
-      set { IncorrectlyNotRejectedParameter.Value.Value = value; }
+    public IntMatrix RejectedStats {
+      get { return RejectedStatsParameter.Value; }
+      set { RejectedStatsParameter.Value = value; }
+    }
+
+    public IntMatrix TotalStats {
+      get { return NotRejectedStatsParameter.Value; }
+      set { NotRejectedStatsParameter.Value = value; }
     }
     #endregion
@@ -106 +90 @@
     public SymbolicRegressionSingleObjectiveOsgaEvaluator() {
       Parameters.Add(new ValueLookupParameter<DoubleValue>("ComparisonFactor", "Determines if the quality should be compared to the better parent (1.0), to the worse (0.0) or to any linearly interpolated value between them."));
-      Parameters.Add(new FixedValueParameter<PercentValue>(RelativeParentChildQualityThresholdParameterName, new PercentValue(0.1)));
+      Parameters.Add(new FixedValueParameter<PercentValue>(RelativeParentChildQualityThresholdParameterName, new PercentValue(0.9)));
       Parameters.Add(new FixedValueParameter<PercentValue>(RelativeFitnessEvaluationIntervalSizeParameterName, new PercentValue(0.1)));
-      Parameters.Add(new FixedValueParameter<IntValue>("CorrectlyRejected", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<IntValue>("IncorrectlyRejected", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<IntValue>("CorrectlyNotRejected", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<IntValue>("IncorrectlyNotRejected", new IntValue(0)));
       Parameters.Add(new LookupParameter<ResultCollection>(ResultCollectionParameterName));
       Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ParentQualities") { ActualName = "Quality" });
+      Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
+      Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
     }
 
@@ -123 +105 @@
       if (!Parameters.ContainsKey("ParentQualities"))
         Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ParentQualities") { ActualName = "Quality" });
+
+      if (!Parameters.ContainsKey("RejectedStats"))
+        Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
+
+      if (!Parameters.ContainsKey("TotalStats"))
+        Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
     }
 
@@ -136 +124 @@
     public override void ClearState() {
       base.ClearState();
-      CorrectlyNotRejected = 0;
-      CorrectlyRejected = 0;
-      IncorrectlyNotRejected = 0;
-      IncorrectlyRejected = 0;
+      RejectedStats = new IntMatrix();
+      TotalStats = new IntMatrix();
     }
 
@@ -216 +202 @@
 
       var interval = (int)Math.Floor(problemData.TrainingPartition.Size * RelativeFitnessEvaluationIntervalSize);
-      var i = 0;
+      var i = problemData.TrainingPartition.Start;
+      var trainingEnd = problemData.TrainingPartition.End;
       var qualityPerInterval = new List<double>();
       while (targetValuesEnumerator.MoveNext() && scaledEstimatedValuesEnumerator.MoveNext()) {
         pearsonRCalculator.Add(targetValuesEnumerator.Current, scaledEstimatedValuesEnumerator.Current);
         ++i;
-        if (i % interval == 0) {
+        if (i % interval == 0 || i == trainingEnd) {
           var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
           qualityPerInterval.Add(q * q);
@@ -235 +222 @@
       var threshold = parentQuality * RelativeParentChildQualityThreshold;
 
-      //var predictedRejected = qualityPerInterval.Any(x => double.IsNaN(x) || !(x > threshold));
-
       bool predictedRejected = false;
-
-      DataTable table;
-      var results = ResultCollectionParameter.ActualValue;
-      if (!results.ContainsKey("RejectionCounts")) {
-        table = new DataTable("RejectionCounts");
-        results.Add(new Result("RejectionCounts", table));
-
-        var row = new DataRow("Predicted Rejected") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.Histogram } };
-        table.Rows.Add(row);
-
-        row = new DataRow("Actually Rejected") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.Histogram } };
-        table.Rows.Add(row);
-
-        //        row = new DataRow("Actually Not Rejected") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.Columns, StartIndexZero = true } };
-        //        row.Values.AddRange(qualityPerInterval.Select(x => 0.0));
-        //        table.Rows.Add(row);
-        //
-        //        row = new DataRow("Predicted Not Rejected") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.Columns, StartIndexZero = true } };
-        //        row.Values.AddRange(qualityPerInterval.Select(x => 0.0));
-        //        table.Rows.Add(row);
-      } else {
-        table = (DataTable)results["RejectionCounts"].Value;
-      }
 
       i = 0;
@@ -272 +234 @@
 
       var actuallyRejected = !(actualQuality > parentQuality);
+
+      if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
+        RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
+        RejectedStats.RowNames = new[] { "Predicted", "Actual" };
+        RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
+        TotalStats = new IntMatrix(2, 2);
+        TotalStats.RowNames = new[] { "Predicted", "Actual" };
+        TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
+      }
+      // gather some statistics
       if (predictedRejected) {
-        table.Rows["Predicted Rejected"].Values.Add(i);
-        if (actuallyRejected)
-          table.Rows["Actually Rejected"].Values.Add(i);
-      }
-      //      else {
-      //        table.Rows["Predicted Not Rejected"].Values[i]++;
-      //        if (!actuallyRejected)
-      //          table.Rows["Actually Not Rejected"].Values[i]++;
-      //      }
-
-      if (predictedRejected) {
-        if (actuallyRejected) {
-          CorrectlyRejected++;
-        } else {
-          IncorrectlyRejected++;
-        }
-      } else {
-        if (actuallyRejected) {
-          IncorrectlyNotRejected++;
-        } else {
-          CorrectlyNotRejected++;
-        }
+        RejectedStats[0, i]++;
+        TotalStats[0, 0]++;
+      } else {
+        TotalStats[0, 1]++;
+      }
+      if (actuallyRejected) {
+        TotalStats[1, 0]++;
+      } else {
+        TotalStats[1, 1]++;
+      }
+      if (predictedRejected && actuallyRejected) {
+        RejectedStats[1, i]++;
       }
       return r * r;