Changeset 14584

Timestamp:

01/18/17 15:40:34 (8 years ago)

Author:

bburlacu

Message:

#2635: Improve reporting of statistics about rejected individuals.

Location:

branches/HeuristicLab.OSGAEvaluator

Files:

• HeuristicLab.OSGAEvaluator.sln (modified) (1 diff)
• HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs (modified) (5 diffs)
• HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs (modified) (9 diffs)

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator.sln

@@ -7 +7 @@
 EndProject
 Global
-  GlobalSection(Performance) = preSolution
-    HasPerformanceSessions = true
-  EndGlobalSection
   GlobalSection(SolutionConfigurationPlatforms) = preSolution
     Debug|Any CPU = Debug|Any CPU

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs

@@ -1 @@
-using System.Linq;
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2017 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
 using HeuristicLab.Analysis;
 using HeuristicLab.Common;
@@ -44 +65 @@
 
       ResultCollection predictionResults;
-      DataTable rejectedStatsTable, totalStatsTable, rejectedStatsPerGenerationTable, nonRejectedStatsPerGenerationTable;
+      DataTable rejectedStatsTable, totalStatsTable, rejectedStatsPerGenerationTable;
       if (!ResultCollection.ContainsKey("OS Prediction")) {
         predictionResults = new ResultCollection();
@@ -58 +79 @@
         predictionResults.Add(new Result("Total Stats", totalStatsTable));
         rejectedStatsPerGenerationTable = new DataTable("Rejected Per Generation");
-        nonRejectedStatsPerGenerationTable = new DataTable("Non Rejected Stats per Generation");
         foreach (var rowName in rejectedStats.RowNames) {
           rejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
           rejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
-          nonRejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
-          nonRejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
         }
         predictionResults.Add(new Result("Rejected Stats Per Generation", rejectedStatsPerGenerationTable));
-        predictionResults.Add(new Result("Non Rejected Stats Per Generation", nonRejectedStatsPerGenerationTable));
         ResultCollection.Add(new Result("OS Prediction", predictionResults));
       } else {
@@ -73 +90 @@
         totalStatsTable = (DataTable)predictionResults["Total Stats"].Value;
         rejectedStatsPerGenerationTable = (DataTable)predictionResults["Rejected Stats Per Generation"].Value;
-        nonRejectedStatsPerGenerationTable = (DataTable)predictionResults["Non Rejected Stats Per Generation"].Value;
       }
 
@@ -88 +104 @@
         var sum = row.Values.Sum();
         row.Values.Add(totalStats[i, 0] - sum);
-        row = nonRejectedStatsPerGenerationTable.Rows[rowName];
-        sum = row.Values.Sum();
-        row.Values.Add(totalStats[i, 1] - sum);
         ++i;
       }

branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

@@ -63 +63 @@
       get { return (IValueParameter<BoolValue>)Parameters[AggregateStatisticsParameterName]; }
     }
-    public IValueParameter<IntMatrix> RejectedStatsParameter {
-      get { return (IValueParameter<IntMatrix>)Parameters["RejectedStats"]; }
-    }
-    public IValueParameter<IntMatrix> NotRejectedStatsParameter {
-      get { return (IValueParameter<IntMatrix>)Parameters["TotalStats"]; }
-    }
     public IValueLookupParameter<DoubleValue> ComparisonFactorParameter {
       get { return (ValueLookupParameter<DoubleValue>)Parameters["ComparisonFactor"]; }
@@ -106 +100 @@
       set { RelativeFitnessEvaluationIntervalSizeParameter.Value.Value = value; }
     }
-    public IntMatrix RejectedStats {
-      get { return RejectedStatsParameter.Value; }
-      set { RejectedStatsParameter.Value = value; }
-    }
-    public IntMatrix TotalStats {
-      get { return NotRejectedStatsParameter.Value; }
-      set { NotRejectedStatsParameter.Value = value; }
-    }
+
     #endregion
 
@@ -120 +107 @@
     }
 
+    // keep track of statistics
     public double AdjustedEvaluatedSolutions { get; set; }
+    public IntMatrix RejectedStats { get; set; }
+    public IntMatrix TotalStats { get; set; }
 
     public SymbolicRegressionSingleObjectiveOsgaEvaluator() {
@@ -128 +118 @@
       Parameters.Add(new LookupParameter<ResultCollection>(ResultCollectionParameterName));
       Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>("ParentQualities") { ActualName = "Quality" });
-      Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
-      Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
       Parameters.Add(new ValueParameter<BoolValue>(AggregateStatisticsParameterName, new BoolValue(false)));
       Parameters.Add(new LookupParameter<DoubleValue>(ActualSelectionPressureParameterName));
@@ -135 +123 @@
       Parameters.Add(new FixedValueParameter<BoolValue>(UseFixedEvaluationIntervalsParameterName, new BoolValue(false)));
       Parameters.Add(new FixedValueParameter<BoolValue>(PreserveResultCompatibilityParameterName, new BoolValue(false)));
+
+      RejectedStats = new IntMatrix();
+      TotalStats = new IntMatrix();
     }
 
@@ -153 +144 @@
 
     [StorableConstructor]
-    protected SymbolicRegressionSingleObjectiveOsgaEvaluator(bool deserializing) : base(deserializing) { }
-
-    protected SymbolicRegressionSingleObjectiveOsgaEvaluator(SymbolicRegressionSingleObjectiveOsgaEvaluator original, Cloner cloner) : base(original, cloner) { }
+    protected SymbolicRegressionSingleObjectiveOsgaEvaluator(bool deserializing) : base(deserializing) {
+      TotalStats = new IntMatrix();
+      RejectedStats = new IntMatrix();
+    }
+
+    protected SymbolicRegressionSingleObjectiveOsgaEvaluator(SymbolicRegressionSingleObjectiveOsgaEvaluator original,
+      Cloner cloner) : base(original, cloner) {
+      if (original.TotalStats != null)
+        TotalStats = cloner.Clone(original.TotalStats);
+
+      if (original.RejectedStats != null)
+        RejectedStats = cloner.Clone(original.RejectedStats);
+    }
 
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -262 +263 @@
             bool predictedRejected = false;
 
-            calculatedRows = 0;
-            foreach (var q in qualityPerInterval) {
+            int i;
+
+            for (i = 0; i < qualityPerInterval.Count; ++i) {
+              var q = qualityPerInterval[i];
               if (double.IsNaN(q) || !(q > threshold)) {
                 predictedRejected = true;
@@ -269 +272 @@
                 break;
               }
-              ++calculatedRows;
             }
 
@@ -275 +277 @@
 
             if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
-              RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
+              RejectedStats = new IntMatrix(2, qualityPerInterval.Count + 1);
               RejectedStats.RowNames = new[] { "Predicted", "Actual" };
               RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
-              TotalStats = new IntMatrix(2, 2);
+              TotalStats = new IntMatrix(2, 1);
               TotalStats.RowNames = new[] { "Predicted", "Actual" };
-              TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
-            }
-            // gather some statistics
-            if (predictedRejected) {
-              RejectedStats[0, calculatedRows]++;
-              TotalStats[0, 0]++;
+              TotalStats.ColumnNames = new[] { "Rejected" };
+            }
+
+            if (actuallyRejected) {
+              TotalStats[0, 0]++; // prediction true
+              TotalStats[1, 0]++;
+              RejectedStats[0, i]++;
+              RejectedStats[1, i]++;
             } else {
-              TotalStats[0, 1]++;
-            }
-            if (actuallyRejected) {
-              TotalStats[1, 0]++;
-            } else {
-              TotalStats[1, 1]++;
-            }
-            if (predictedRejected && actuallyRejected) {
-              RejectedStats[1, calculatedRows]++;
+              if (predictedRejected) {
+                RejectedStats[0, i]++;
+                TotalStats[0, 0]++;
+              }
             }
           }