Changeset 14231


Ignore:
Timestamp:
08/03/16 18:26:24 (3 years ago)
Author:
bburlacu
Message:

#2635: Add evaluator option to aggregate statistics during evaluation (slower). Improve analyzer.

Location:
branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/OSGAPredictionCountsAnalyzer.cs

    r14104 r14231  
    4444
    4545      ResultCollection predictionResults;
    46       DataTable rejectedStatsTable, totalStatsTable;
     46      DataTable rejectedStatsTable, totalStatsTable, rejectedStatsPerGenerationTable, nonRejectedStatsPerGenerationTable;
    4747      if (!ResultCollection.ContainsKey("OS Prediction")) {
    4848        predictionResults = new ResultCollection();
     
    5757        }
    5858        predictionResults.Add(new Result("Total Stats", totalStatsTable));
    59 
     59        rejectedStatsPerGenerationTable = new DataTable("Rejected Per Generation");
     60        nonRejectedStatsPerGenerationTable = new DataTable("Non Rejected Stats per Generation");
     61        foreach (var rowName in rejectedStats.RowNames) {
     62          rejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
     63          rejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
     64          nonRejectedStatsPerGenerationTable.Rows.Add(new DataRow(rowName) { VisualProperties = { StartIndexZero = true, ChartType = DataRowVisualProperties.DataRowChartType.Line, LineWidth = 1 } });
     65          nonRejectedStatsPerGenerationTable.Rows[rowName].Values.Add(0d);
     66        }
     67        predictionResults.Add(new Result("Rejected Stats Per Generation", rejectedStatsPerGenerationTable));
     68        predictionResults.Add(new Result("Non Rejected Stats Per Generation", nonRejectedStatsPerGenerationTable));
    6069        ResultCollection.Add(new Result("OS Prediction", predictionResults));
    6170      } else {
     
    6372        rejectedStatsTable = (DataTable)predictionResults["Rejected Stats"].Value;
    6473        totalStatsTable = (DataTable)predictionResults["Total Stats"].Value;
     74        rejectedStatsPerGenerationTable = (DataTable)predictionResults["Rejected Stats Per Generation"].Value;
     75        nonRejectedStatsPerGenerationTable = (DataTable)predictionResults["Non Rejected Stats Per Generation"].Value;
    6576      }
    6677
    6778      int i = 0;
    6879      foreach (var rowName in rejectedStats.RowNames) {
    69         // add a padding zero below to prevent columns from being cut off to the left
     80        // pad values with a 0 on each side to prevent clipping
    7081        rejectedStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, rejectedStats.Columns).Select(j => (double)rejectedStats[i, j])).Concat(new[] { 0d }));
    7182        ++i;
     
    7485      foreach (var rowName in totalStats.RowNames) {
    7586        totalStatsTable.Rows[rowName].Values.Replace(new[] { 0d }.Concat(Enumerable.Range(0, totalStats.Columns).Select(j => (double)totalStats[i, j])).Concat(new[] { 0d }));
     87        var row = rejectedStatsPerGenerationTable.Rows[rowName];
     88        var sum = row.Values.Sum();
     89        row.Values.Add(totalStats[i, 0] - sum);
     90        row = nonRejectedStatsPerGenerationTable.Rows[rowName];
     91        sum = row.Values.Sum();
     92        row.Values.Add(totalStats[i, 1] - sum);
    7693        ++i;
    7794      }
  • branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

    r14184 r14231  
    3838    private const string RelativeFitnessEvaluationIntervalSizeParameterName = "RelativeFitnessEvaluationIntervalSize";
    3939    private const string ResultCollectionParameterName = "Results";
     40    private const string AggregateStatisticsParameterName = "AggregateStatistics";
    4041
    4142    #region parameters
     
    4344      get { return (ILookupParameter<ResultCollection>)Parameters[ResultCollectionParameterName]; }
    4445    }
    45 
     46    public IValueParameter<BoolValue> AggregateStatisticsParameter {
     47      get { return (IValueParameter<BoolValue>)Parameters[AggregateStatisticsParameterName]; }
     48    }
    4649    public IValueParameter<IntMatrix> RejectedStatsParameter {
    4750      get { return (IValueParameter<IntMatrix>)Parameters["RejectedStats"]; }
     
    9699      Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
    97100      Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
     101      Parameters.Add(new ValueParameter<BoolValue>(AggregateStatisticsParameterName, new BoolValue(false)));
    98102    }
    99103
     
    184188      var targetValuesEnumerator = targetValues.GetEnumerator();
    185189      var estimatedValuesEnumerator = estimatedValues.GetEnumerator();
    186 
    187       var i = 0;
    188190      var trainingPartitionSize = problemData.TrainingPartition.Size;
    189191      var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
    190       while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
    191         pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
    192         ++i;
    193         if (i % interval == 0 || i == trainingPartitionSize) {
    194           var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
    195           var quality = q * q;
    196           if (!(quality > threshold))
    197             return quality;
    198         }
     192
     193      var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
     194      var i = 0;
     195      if (aggregateStatistics) {
     196        var trainingEnd = problemData.TrainingPartition.End;
     197        var qualityPerInterval = new List<double>();
     198        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
     199          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
     200          ++i;
     201          if (i % interval == 0 || i == trainingPartitionSize) {
     202            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     203            qualityPerInterval.Add(q * q);
     204          }
     205        }
     206        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     207        var actualQuality = r * r;
     208
     209        bool predictedRejected = false;
     210
     211        i = 0;
     212        double quality = actualQuality;
     213        foreach (var q in qualityPerInterval) {
     214          if (double.IsNaN(q) || !(q > threshold)) {
     215            predictedRejected = true;
     216            quality = q;
     217            break;
     218          }
     219          ++i;
     220        }
     221
     222        var actuallyRejected = !(actualQuality > parentQuality);
     223
     224        if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
     225          RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
     226          RejectedStats.RowNames = new[] { "Predicted", "Actual" };
     227          RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
     228          TotalStats = new IntMatrix(2, 2);
     229          TotalStats.RowNames = new[] { "Predicted", "Actual" };
     230          TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
     231        }
     232        // gather some statistics
     233        if (predictedRejected) {
     234          RejectedStats[0, i]++;
     235          TotalStats[0, 0]++;
     236        } else {
     237          TotalStats[0, 1]++;
     238        }
     239        if (actuallyRejected) {
     240          TotalStats[1, 0]++;
     241        } else {
     242          TotalStats[1, 1]++;
     243        }
     244        if (predictedRejected && actuallyRejected) {
     245          RejectedStats[1, i]++;
     246        }
     247        return quality;
     248      } else {
     249        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
     250          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
     251          ++i;
     252          if (i % interval == 0 || i == trainingPartitionSize) {
     253            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     254            var quality = q * q;
     255            if (!(quality > threshold))
     256              return quality;
     257          }
     258        }
     259        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     260        var actualQuality = r * r;
     261        return actualQuality;
    199262      }
    200       var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
    201       var actualQuality = r * r;
    202       return actualQuality;
    203263    }
    204264
Note: See TracChangeset for help on using the changeset viewer.