Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
08/03/16 18:26:24 (8 years ago)
Author:
bburlacu
Message:

#2635: Add evaluator option to aggregate statistics during evaluation (slower). Improve analyzer.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/HeuristicLab.OSGAEvaluator/HeuristicLab.OSGAEvaluator/SymbolicRegressionSingleObjectiveOSGAEvaluator.cs

    r14184 r14231  
    3838    private const string RelativeFitnessEvaluationIntervalSizeParameterName = "RelativeFitnessEvaluationIntervalSize";
    3939    private const string ResultCollectionParameterName = "Results";
     40    private const string AggregateStatisticsParameterName = "AggregateStatistics";
    4041
    4142    #region parameters
     
    4344      get { return (ILookupParameter<ResultCollection>)Parameters[ResultCollectionParameterName]; }
    4445    }
    45 
     46    public IValueParameter<BoolValue> AggregateStatisticsParameter {
     47      get { return (IValueParameter<BoolValue>)Parameters[AggregateStatisticsParameterName]; }
     48    }
    4649    public IValueParameter<IntMatrix> RejectedStatsParameter {
    4750      get { return (IValueParameter<IntMatrix>)Parameters["RejectedStats"]; }
     
    9699      Parameters.Add(new ValueParameter<IntMatrix>("RejectedStats", new IntMatrix()));
    97100      Parameters.Add(new ValueParameter<IntMatrix>("TotalStats", new IntMatrix()));
     101      Parameters.Add(new ValueParameter<BoolValue>(AggregateStatisticsParameterName, new BoolValue(false)));
    98102    }
    99103
     
    184188      var targetValuesEnumerator = targetValues.GetEnumerator();
    185189      var estimatedValuesEnumerator = estimatedValues.GetEnumerator();
    186 
    187       var i = 0;
    188190      var trainingPartitionSize = problemData.TrainingPartition.Size;
    189191      var interval = (int)Math.Floor(trainingPartitionSize * RelativeFitnessEvaluationIntervalSize);
    190       while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
    191         pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
    192         ++i;
    193         if (i % interval == 0 || i == trainingPartitionSize) {
    194           var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
    195           var quality = q * q;
    196           if (!(quality > threshold))
    197             return quality;
    198         }
     192
     193      var aggregateStatistics = AggregateStatisticsParameter.Value.Value;
     194      var i = 0;
     195      if (aggregateStatistics) {
     196        var trainingEnd = problemData.TrainingPartition.End;
     197        var qualityPerInterval = new List<double>();
     198        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
     199          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
     200          ++i;
     201          if (i % interval == 0 || i == trainingPartitionSize) {
     202            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     203            qualityPerInterval.Add(q * q);
     204          }
     205        }
     206        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     207        var actualQuality = r * r;
     208
     209        bool predictedRejected = false;
     210
     211        i = 0;
     212        double quality = actualQuality;
     213        foreach (var q in qualityPerInterval) {
     214          if (double.IsNaN(q) || !(q > threshold)) {
     215            predictedRejected = true;
     216            quality = q;
     217            break;
     218          }
     219          ++i;
     220        }
     221
     222        var actuallyRejected = !(actualQuality > parentQuality);
     223
     224        if (RejectedStats.Rows == 0 || TotalStats.Rows == 0) {
     225          RejectedStats = new IntMatrix(2, qualityPerInterval.Count);
     226          RejectedStats.RowNames = new[] { "Predicted", "Actual" };
     227          RejectedStats.ColumnNames = Enumerable.Range(1, RejectedStats.Columns).Select(x => string.Format("0-{0}", Math.Min(trainingEnd, x * interval)));
     228          TotalStats = new IntMatrix(2, 2);
     229          TotalStats.RowNames = new[] { "Predicted", "Actual" };
     230          TotalStats.ColumnNames = new[] { "Rejected", "Not Rejected" };
     231        }
     232        // gather some statistics
     233        if (predictedRejected) {
     234          RejectedStats[0, i]++;
     235          TotalStats[0, 0]++;
     236        } else {
     237          TotalStats[0, 1]++;
     238        }
     239        if (actuallyRejected) {
     240          TotalStats[1, 0]++;
     241        } else {
     242          TotalStats[1, 1]++;
     243        }
     244        if (predictedRejected && actuallyRejected) {
     245          RejectedStats[1, i]++;
     246        }
     247        return quality;
     248      } else {
     249        while (targetValuesEnumerator.MoveNext() && estimatedValuesEnumerator.MoveNext()) {
     250          pearsonRCalculator.Add(targetValuesEnumerator.Current, estimatedValuesEnumerator.Current);
     251          ++i;
     252          if (i % interval == 0 || i == trainingPartitionSize) {
     253            var q = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     254            var quality = q * q;
     255            if (!(quality > threshold))
     256              return quality;
     257          }
     258        }
     259        var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
     260        var actualQuality = r * r;
     261        return actualQuality;
    199262      }
    200       var r = pearsonRCalculator.ErrorState != OnlineCalculatorError.None ? double.NaN : pearsonRCalculator.R;
    201       var actualQuality = r * r;
    202       return actualQuality;
    203263    }
    204264
Note: See TracChangeset for help on using the changeset viewer.