Free cookie consent management tool by TermsFeed Policy Generator

Changeset 482


Ignore:
Timestamp:
08/10/08 12:44:27 (16 years ago)
Author:
gkronber
Message:

made a few more improvements in the GP evaluators (ticket #242 All GP evaluators should support the 'UseEstimatedTargetValues' switch for autoregressive modelling)

Location:
trunk/sources/HeuristicLab.StructureIdentification/Evaluation
Files:
11 edited

Legend:

Unmodified
Added
Removed

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/AccuracyEvaluator.cs

    r480 r482  
    3535    private double[] classesArr;
    3636    private double[] thresholds;
     37    private DoubleData accuracy;
    3738    public override string Description {
    3839      get {
    39         return @"TASK";
     40        return @"Calculates the total accuracy of the model (ratio of correctly classified instances to total number of instances) given a model and the list of possible target class values.";
    4041      }
    4142    }
    … …  
    4344    public AccuracyEvaluator()
    4445      : base() {
     46      AddVariableInfo(new VariableInfo("Accuracy", "The total accuracy of the model (ratio of correctly classified instances to total number of instances)", typeof(DoubleData), VariableKind.New));
    4547      AddVariableInfo(new VariableInfo("TargetClassValues", "The original class values of target variable (for instance negative=0 and positive=1).", typeof(ItemList<DoubleData>), VariableKind.In));
    4648    }
    4749
    4850    public override IOperation Apply(IScope scope) {
     51      accuracy = GetVariableValue<DoubleData>("Accuracy", scope, false, false);
     52      if(accuracy == null) {
     53        accuracy = new DoubleData();
     54        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("Accuracy"), accuracy));
     55      }
     56
    4957      ItemList<DoubleData> classes = GetVariableValue<ItemList<DoubleData>>("TargetClassValues", scope, true);
    5058      classesArr = new double[classes.Count];
    … …  
    5967    }
    6068
    61     public override double Evaluate(int start, int end) {
    62       int nSamples = end-start;
     69    public override void Evaluate(int start, int end) {
     70      int nSamples = end - start;
    6371      int nCorrect = 0;
    6472      for(int sample = start; sample < end; sample++) {
    … …  
    7078        if(est < thresholds[0]) estClass = classesArr[0];
    7179        // if estimation is larger (or equal) than the largest threshold value -> estimated class is the upper class
    72         else if(est >= thresholds[thresholds.Length - 1]) estClass = classesArr[classesArr.Length - 1]; 
     80        else if(est >= thresholds[thresholds.Length - 1]) estClass = classesArr[classesArr.Length - 1];
    7381        else {
    7482          // otherwise the estimated class is the class which upper threshold is larger than the estimated value
    … …  
    8290        if(Math.Abs(estClass - origClass) < EPSILON) nCorrect++;
    8391      }
    84       return nCorrect / (double)nSamples;
     92      accuracy.Data = nCorrect / (double)nSamples;
    8593    }
    8694  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/ClassificationMeanSquaredErrorEvaluator.cs

    r480 r482  
    3131
    3232namespace HeuristicLab.StructureIdentification {
    33   public class ClassificationMeanSquaredErrorEvaluator : GPEvaluatorBase {
     33  public class ClassificationMeanSquaredErrorEvaluator : MeanSquaredErrorEvaluator {
    3434    private const double EPSILON = 1.0E-6;
    3535    private double[] classesArr;
    … …  
    5454    }
    5555
    56     public override double Evaluate(int start, int end) {
    57 
     56    public override void Evaluate(int start, int end) {
    5857      double errorsSquaredSum = 0;
    5958      for(int sample = start; sample < end; sample++) {
    … …  
    7978        errorsSquaredSum = double.MaxValue;
    8079      }
    81       return errorsSquaredSum;
     80      mse.Data = errorsSquaredSum;
    8281    }
    8382

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/CoefficientOfDeterminationEvaluator.cs

    r480 r482  
    3232namespace HeuristicLab.StructureIdentification {
    3333  public class CoefficientOfDeterminationEvaluator : GPEvaluatorBase {
     34    private DoubleData r2;
    3435    public override string Description {
    3536      get {
    … …  
    4142    public CoefficientOfDeterminationEvaluator()
    4243      : base() {
     44      AddVariableInfo(new VariableInfo("R2", "The coefficient of determination of the model", typeof(DoubleData), VariableKind.New));
    4345    }
    4446
    45     public override double Evaluate(int start, int end) {
     47    public override IOperation Apply(IScope scope) {
     48      r2 = GetVariableValue<DoubleData>("R2", scope, false, false);
     49      if(r2 == null) {
     50        r2 = new DoubleData();
     51        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("R2"), r2));
     52      }
     53
     54      return base.Apply(scope);
     55    }
     56
     57    public override void Evaluate(int start, int end) {
    4658      double errorsSquaredSum = 0.0;
    4759      double originalDeviationTotalSumOfSquares = 0.0;
    … …  
    6072
    6173      double quality = 1 - errorsSquaredSum / originalDeviationTotalSumOfSquares;
    62       if(quality > 1) 
     74      if(quality > 1)
    6375        throw new InvalidProgramException();
    64       if(double.IsNaN(quality) || double.IsInfinity(quality)) 
     76      if(double.IsNaN(quality) || double.IsInfinity(quality))
    6577        quality = double.MaxValue;
    66       return quality;
     78      r2.Data = quality;
    6779    }
    6880  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/EarlyStoppingMeanSquaredErrorEvaluator.cs

    r480 r482  
    5252
    5353    // evaluates the function-tree for the given target-variable and the whole dataset and returns the MSE
    54     public override double Evaluate(int start, int end) {
     54    public override void Evaluate(int start, int end) {
    5555      double errorsSquaredSum = 0;
    5656      int rows = end - start;
    … …  
    6565        // check the limit and stop as soon as we hit the limit
    6666        if(errorsSquaredSum / rows >= qualityLimit) {
    67           return errorsSquaredSum / (sample - start + 1); // return estimated MSE (when the remaining errors are on average the same)
     67          mse.Data = errorsSquaredSum / (sample - start + 1); // return estimated MSE (when the remaining errors are on average the same)
     68          return;
    6869        }
    6970      }
    … …  
    7273        errorsSquaredSum = double.MaxValue;
    7374      }
    74       return errorsSquaredSum;
     75      mse.Data = errorsSquaredSum;
    7576    }
    7677  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/GPEvaluatorBase.cs

    r481 r482  
    3434    private IEvaluator evaluator;
    3535    private int targetVariable;
    36     private int trainingStart;
    37     private int trainingEnd;
     36    private int start;
     37    private int end;
    3838    private bool useEstimatedValues;
    3939    private double[] backupValues;
    … …  
    5555      AddVariableInfo(new VariableInfo("PunishmentFactor", "Punishment factor for invalid estimations", typeof(DoubleData), VariableKind.In));
    5656      AddVariableInfo(new VariableInfo("TotalEvaluatedNodes", "Number of evaluated nodes", typeof(DoubleData), VariableKind.In | VariableKind.Out));
    57       AddVariableInfo(new VariableInfo("TrainingSamplesStart", "Start index of training samples in dataset", typeof(IntData), VariableKind.In));
    58       AddVariableInfo(new VariableInfo("TrainingSamplesEnd", "End index of training samples in dataset", typeof(IntData), VariableKind.In));
     57      AddVariableInfo(new VariableInfo("SamplesStart", "Start index of samples in dataset to evaluate", typeof(IntData), VariableKind.In));
     58      AddVariableInfo(new VariableInfo("SamplesEnd", "End index of samples in dataset to evaluate", typeof(IntData), VariableKind.In));
    5959      AddVariableInfo(new VariableInfo("UseEstimatedTargetValue", "Wether to use the original (measured) or the estimated (calculated) value for the targat variable when doing autoregressive modelling", typeof(BoolData), VariableKind.In));
    60       AddVariableInfo(new VariableInfo("Quality", "The evaluated quality of the model", typeof(DoubleData), VariableKind.New));
    6160    }
    6261
    … …  
    6968      treeSize = scope.GetVariableValue<IntData>("TreeSize", false).Data;
    7069      totalEvaluatedNodes = scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data;
    71       int trainingStart = GetVariableValue<IntData>("TrainingSamplesStart", scope, true).Data;
    72       int trainingEnd = GetVariableValue<IntData>("TrainingSamplesEnd", scope, true).Data;
     70      int start = GetVariableValue<IntData>("SamplesStart", scope, true).Data;
     71      int end = GetVariableValue<IntData>("SamplesEnd", scope, true).Data;
    7372      useEstimatedValues = GetVariableValue<BoolData>("UseEstimatedTargetValue", scope, true).Data;
    7473      // prepare for autoregressive modelling by saving the original values of the target-variable to a backup array
    7574      if(useEstimatedValues &&
    76         (backupValues == null || trainingStart!=this.trainingStart || trainingEnd!=this.trainingEnd)) {
    77         this.trainingStart = trainingStart;
    78         this.trainingEnd = trainingEnd;
    79         backupValues = new double[trainingEnd - trainingStart];
    80         for(int i = trainingStart; i < trainingEnd; i++) {
    81           backupValues[i - trainingStart] = dataset.GetValue(i, targetVariable);
     75        (backupValues == null || start!=this.start || end!=this.end)) {
     76        this.start = start;
     77        this.end = end;
     78        backupValues = new double[end - start];
     79        for(int i = start; i < end; i++) {
     80          backupValues[i - start] = dataset.GetValue(i, targetVariable);
    8281        }
    8382      }
    8483      // get the mean of the values of the target variable to determin the max and min bounds of the estimated value
    85       targetMean = dataset.GetMean(targetVariable, trainingStart, trainingEnd);
     84      targetMean = dataset.GetMean(targetVariable, start, end);
    8685      estimatedValueMin = targetMean - maximumPunishment;
    8786      estimatedValueMax = targetMean + maximumPunishment;
    … …  
    9291      evaluatedSamples = 0;
    9392
    94       // calculate the quality measure
    95       double result = Evaluate(trainingStart, trainingEnd);
     93      Evaluate(start, end);
    9694
    9795      // restore the values of the target variable from the backup array if necessary
    98       if(useEstimatedValues) RestoreDataset(dataset, targetVariable, trainingStart, trainingEnd);
     96      if(useEstimatedValues) RestoreDataset(dataset, targetVariable, start, end);
    9997      // update the value of total evaluated nodes
    10098      scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data = totalEvaluatedNodes + treeSize * evaluatedSamples;
    10199      // write the calculate quality value
    102       DoubleData quality = GetVariableValue<DoubleData>("Quality", scope, false, false);
    103       if(quality == null) {
    104         scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("Quality"), new DoubleData(result)));
    105       } else {
    106         quality.Data = result;
    107       }
    108100      return null;
    109101    }
    … …  
    115107    }
    116108
    117     public abstract double Evaluate(int start, int end);
     109    public abstract void Evaluate(int start, int end);
    118110
    119111    public void SetOriginalValue(int sample, double value) {

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MCCEvaluator.cs

    r480 r482  
    3535    private double[] original = new double[1];
    3636    private double[] estimated = new double[1];
     37    private DoubleData mcc;
    3738    public override string Description {
    3839      get {
    39         return @"TASK";
     40        return @"Calculates the matthews correlation coefficient for a given model and class separation threshold";
    4041      }
    4142    }
    … …  
    4344      : base() {
    4445      AddVariableInfo(new VariableInfo("ClassSeparation", "The value of separation between negative and positive target classification values (for instance 0.5 if negative=0 and positive=1).", typeof(DoubleData), VariableKind.In));
     46      AddVariableInfo(new VariableInfo("MCC", "The matthews correlation coefficient of the model", typeof(DoubleData), VariableKind.New));
    4547    }
    4648
    4749    public override IOperation Apply(IScope scope) {
     50      mcc = GetVariableValue<DoubleData>("MCC", scope, false, false);
     51      if(mcc == null) {
     52        mcc = new DoubleData();
     53        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("MCC"), mcc));
     54      }
    4855      limit = GetVariableValue<DoubleData>("ClassSeparation", scope, true).Data;
    4956      return base.Apply(scope);
    5057    }
    5158
    52     public override double Evaluate(int start, int end) {
     59    public override void Evaluate(int start, int end) {
    5360      int nSamples = end - start;
    5461      if(estimated.Length != nSamples) {
    … …  
    8592        }
    8693      }
    87       return best_mcc;
     94      this.mcc.Data = best_mcc;
    8895    }
    8996  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MeanAbsolutePercentageErrorEvaluator.cs

    r480 r482  
    3232namespace HeuristicLab.StructureIdentification {
    3333  public class MeanAbsolutePercentageErrorEvaluator : GPEvaluatorBase {
     34    private DoubleData mape;
    3435    public override string Description {
    3536      get {
    … …  
    4142    public MeanAbsolutePercentageErrorEvaluator()
    4243      : base() {
     44      AddVariableInfo(new VariableInfo("MAPE", "The mean absolute percentage error of the model", typeof(DoubleData), VariableKind.New));
    4345    }
    4446
    45     public override double Evaluate(int start, int end) {
     47    public override IOperation Apply(IScope scope) {
     48      mape = GetVariableValue<DoubleData>("MAPE", scope, false, false);
     49      if(mape == null) {
     50        mape = new DoubleData();
     51        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("MAPE"), mape));
     52      }
     53
     54      return base.Apply(scope);
     55    }
     56
     57    public override void Evaluate(int start, int end) {
    4658      double errorsSum = 0.0;
    4759      for(int sample = start; sample < end; sample++) {
    … …  
    5769      if(double.IsNaN(quality) || double.IsInfinity(quality))
    5870        quality = double.MaxValue;
    59       return quality;
     71      mape.Data = quality;
    6072    }
    6173  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MeanSquaredErrorEvaluator.cs

    r480 r482  
    3232namespace HeuristicLab.StructureIdentification {
    3333  public class MeanSquaredErrorEvaluator : GPEvaluatorBase {
     34    protected DoubleData mse;
    3435    public override string Description {
    3536      get {
    … …  
    4142    public MeanSquaredErrorEvaluator()
    4243      : base() {
     44      AddVariableInfo(new VariableInfo("MSE", "The mean squared error of the model", typeof(DoubleData), VariableKind.New));
    4345    }
    4446
    45     public override double Evaluate(int start, int end) {
     47    public override IOperation Apply(IScope scope) {
     48      mse = GetVariableValue<DoubleData>("MSE", scope, false, false);
     49      if(mse == null) {
     50        mse = new DoubleData();
     51        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("MSE"), mse));
     52      }
     53
     54      return base.Apply(scope);
     55    }
     56
     57    public override void Evaluate(int start, int end) {
    4658      double errorsSquaredSum = 0;
    4759      for(int sample = start; sample < end; sample++) {
    … …  
    5971        errorsSquaredSum = double.MaxValue;
    6072      }
    61       return errorsSquaredSum;
     73      mse.Data = errorsSquaredSum;
    6274    }
    6375  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/SimpleEvaluator.cs

    r480 r482  
    5252    }
    5353
    54     public override double Evaluate(int start, int end) {
     54    public override void Evaluate(int start, int end) {
    5555      for(int sample = start; sample < end; sample++) {
    5656        ItemList row = new ItemList();
    … …  
    6262        values.Add(row);
    6363      }
    64       return double.NaN;
    6564    }
    6665  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/TheilInequalityCoefficientEvaluator.cs

    r480 r482  
    3333  public class TheilInequalityCoefficientEvaluator : GPEvaluatorBase {
    3434    private bool differential;
     35    private DoubleData theilInequaliy;
    3536    public override string Description {
    3637      get {
    … …  
    4344      : base() {
    4445      AddVariableInfo(new VariableInfo("Differential", "Wether to calculate the coefficient for the predicted change vs. original change or for the absolute prediction vs. original value", typeof(BoolData), VariableKind.In));
     46      AddVariableInfo(new VariableInfo("TheilInequalityCoefficient", "Theil's inequality coefficient of the model", typeof(DoubleData), VariableKind.New));
     47
    4548    }
    4649
    4750    public override IOperation Apply(IScope scope) {
    4851      differential = GetVariableValue<BoolData>("Differential", scope, true).Data;
     52      theilInequaliy = GetVariableValue<DoubleData>("TheilInequalityCoefficient", scope, false, false);
     53      if(theilInequaliy == null) {
     54        theilInequaliy = new DoubleData();
     55        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("TheilInequalityCoefficient"), theilInequaliy));
     56      }
     57
    4958      return base.Apply(scope);
    5059    }
    5160
    52     public override double Evaluate(int start, int end) {
     61    public override void Evaluate(int start, int end) {
    5362      double errorsSquaredSum = 0.0;
    5463      double estimatedSquaredSum = 0.0;
    … …  
    5968        double estimatedChange = GetEstimatedValue(sample) - prevValue;
    6069        double originalChange = GetOriginalValue(sample) - prevValue;
    61         SetOriginalValue(sample, estimatedChange+prevValue);
     70        SetOriginalValue(sample, estimatedChange + prevValue);
    6271        if(!double.IsNaN(originalChange) && !double.IsInfinity(originalChange)) {
    6372          double error = estimatedChange - originalChange;
    … …  
    7180      if(double.IsNaN(quality) || double.IsInfinity(quality))
    7281        quality = double.MaxValue;
    73       return quality;
     82      theilInequaliy.Data = quality;
    7483    }
    7584  }

  • trunk/sources/HeuristicLab.StructureIdentification/Evaluation/VarianceAccountedForEvaluator.cs

    r480 r482  
    3232namespace HeuristicLab.StructureIdentification {
    3333  public class VarianceAccountedForEvaluator : GPEvaluatorBase {
     34    private DoubleData vaf;
    3435    public override string Description {
    3536      get {
    … …  
    5051    public VarianceAccountedForEvaluator()
    5152      : base() {
     53      AddVariableInfo(new VariableInfo("VAF", "The variance-accounted-for quality of the model", typeof(DoubleData), VariableKind.New));
     54
    5255    }
    5356
     57    public override IOperation Apply(IScope scope) {
     58      vaf = GetVariableValue<DoubleData>("VAF", scope, false, false);
     59      if(vaf == null) {
     60        vaf = new DoubleData();
     61        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("VAF"), vaf));
     62      }
    5463
    55     public override double Evaluate(int start, int end) {
     64      return base.Apply(scope);
     65    }
     66
     67    public override void Evaluate(int start, int end) {
    5668      int nSamples = end - start;
    5769      double[] errors = new double[nSamples];
    … …  
    7385        quality = double.MaxValue;
    7486      }
    75       return quality;
     87      vaf.Data = quality;
    7688    }
    7789  }
Note: See TracChangeset for help on using the changeset viewer.