Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
10/16/12 09:44:07 (12 years ago)
Author:
sforsten
Message:

#1776:

Location:
branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis
Files:
19 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs

    r8534 r8811  
    2828using HeuristicLab.Data;
    2929using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    30 using HeuristicLab.Problems.DataAnalysis.Interfaces;
    3130
    3231namespace HeuristicLab.Problems.DataAnalysis {
    … …  
    3736  [Item("Classification Ensemble Solution", "A classification solution that contains an ensemble of multiple classification models")]
    3837  [Creatable("Data Analysis - Ensembles")]
    39   public sealed class ClassificationEnsembleSolution : ClassificationSolution, IClassificationEnsembleSolution {
     38  public sealed class ClassificationEnsembleSolution : ClassificationSolutionBase, IClassificationEnsembleSolution {
     39    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
     40    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
     41    private readonly Dictionary<int, double> evaluationCache = new Dictionary<int, double>();
     42
    4043    public new IClassificationEnsembleModel Model {
    4144      get { return (IClassificationEnsembleModel)base.Model; }
    … …  
    162165    }
    163166
    164     protected override void RecalculateResults() {
    165       weightCalculator.CalculateNormalizedWeights(classificationSolutions.CheckedItems);
    166       CalculateResults();
    167     }
    168167
    169168    #region Evaluation
     169    public override IEnumerable<double> EstimatedClassValues {
     170      get { return GetEstimatedClassValues(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
     171    }
     172
    170173    public override IEnumerable<double> EstimatedTrainingClassValues {
    171174      get {
    … …  
    258261    }
    259262
     263    protected override void RecalculateResults() {
     264      weightCalculator.CalculateNormalizedWeights(classificationSolutions.CheckedItems);
     265      base.RecalculateResults();
     266    }
     267
    260268    private void AddClassificationSolution(IClassificationSolution solution) {
    261269      if (Model.Models.Contains(solution.Model)) throw new ArgumentException();

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

    r8534 r8811  
    223223    }
    224224
    225     private List<double> classValues;
    226     public List<double> ClassValues {
     225    private List<double> classValuesCache;
     226    private List<double> ClassValuesCache {
    227227      get {
    228         if (classValues == null) {
    229           classValues = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().ToList();
    230           classValues.Sort();
     228        if (classValuesCache == null) {
     229          classValuesCache = Dataset.GetDoubleValues(TargetVariableParameter.Value.Value).Distinct().OrderBy(x => x).ToList();
    231230        }
    232         return classValues;
     231        return classValuesCache;
    233232      }
    234233    }
    235     IEnumerable<double> IClassificationProblemData.ClassValues {
    236       get { return ClassValues; }
    237     }
    238 
     234    public IEnumerable<double> ClassValues {
     235      get { return ClassValuesCache; }
     236    }
    239237    public int Classes {
    240       get { return ClassValues.Count; }
    241     }
    242 
    243     private List<string> classNames;
    244     public List<string> ClassNames {
     238      get { return ClassValuesCache.Count; }
     239    }
     240
     241    private List<string> classNamesCache;
     242    private List<string> ClassNamesCache {
    245243      get {
    246         if (classNames == null) {
    247           classNames = new List<string>();
     244        if (classNamesCache == null) {
     245          classNamesCache = new List<string>();
    248246          for (int i = 0; i < ClassNamesParameter.Value.Rows; i++)
    249             classNames.Add(ClassNamesParameter.Value[i, 0]);
     247            classNamesCache.Add(ClassNamesParameter.Value[i, 0]);
    250248        }
    251         return classNames;
     249        return classNamesCache;
    252250      }
    253251    }
    254     IEnumerable<string> IClassificationProblemData.ClassNames {
    255       get { return ClassNames; }
    256     }
    257 
    258     private Dictionary<Tuple<double, double>, double> classificationPenaltiesCache = new Dictionary<Tuple<double, double>, double>();
     252    public IEnumerable<string> ClassNames {
     253      get { return ClassNamesCache; }
     254    }
    259255    #endregion
    260256
    … …  
    284280      TestPartition.Start = classificationProblemData.TestPartition.Start;
    285281      TestPartition.End = classificationProblemData.TestPartition.End;
     282
     283      for (int i = 0; i < classificationProblemData.ClassNames.Count(); i++)
     284        ClassNamesParameter.Value[i, 0] = classificationProblemData.ClassNames.ElementAt(i);
     285
     286      for (int i = 0; i < Classes; i++) {
     287        for (int j = 0; j < Classes; j++) {
     288          ClassificationPenaltiesParameter.Value[i, j] = classificationProblemData.GetClassificationPenalty(ClassValuesCache[i], ClassValuesCache[j]);
     289        }
     290      }
    286291    }
    287292
    … …  
    295300      Parameters.Add(new FixedValueParameter<DoubleMatrix>(ClassificationPenaltiesParameterName, ""));
    296301
     302      RegisterParameterEvents();
    297303      ResetTargetVariableDependentMembers();
    298       RegisterParameterEvents();
    299304    }
    300305
    … …  
    319324      DeregisterParameterEvents();
    320325
    321       classNames = null;
    322326      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Columns = 1;
    323       ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValues.Count;
     327      ((IStringConvertibleMatrix)ClassNamesParameter.Value).Rows = ClassValuesCache.Count;
    324328      for (int i = 0; i < Classes; i++)
    325         ClassNamesParameter.Value[i, 0] = "Class " + ClassValues[i];
     329        ClassNamesParameter.Value[i, 0] = "Class " + ClassValuesCache[i];
    326330      ClassNamesParameter.Value.ColumnNames = new List<string>() { "ClassNames" };
    327331      ClassNamesParameter.Value.RowNames = ClassValues.Select(s => "ClassValue: " + s);
    328332
    329       classificationPenaltiesCache.Clear();
    330       ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
    331333      ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Rows = Classes;
    332334      ((IStringConvertibleMatrix)ClassificationPenaltiesParameter.Value).Columns = Classes;
    … …  
    339341        }
    340342      }
    341       ((ValueParameter<DoubleMatrix>)ClassificationPenaltiesParameter).ReactOnValueToStringChangedAndValueItemImageChanged = true;
    342343      RegisterParameterEvents();
    343344    }
    344345
    345346    public string GetClassName(double classValue) {
    346       if (!ClassValues.Contains(classValue)) throw new ArgumentException();
    347       int index = ClassValues.IndexOf(classValue);
    348       return ClassNames[index];
     347      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
     348      int index = ClassValuesCache.IndexOf(classValue);
     349      return ClassNamesCache[index];
    349350    }
    350351    public double GetClassValue(string className) {
    351       if (!ClassNames.Contains(className)) throw new ArgumentException();
    352       int index = ClassNames.IndexOf(className);
    353       return ClassValues[index];
     352      if (!ClassNamesCache.Contains(className)) throw new ArgumentException();
     353      int index = ClassNamesCache.IndexOf(className);
     354      return ClassValuesCache[index];
    354355    }
    355356    public void SetClassName(double classValue, string className) {
    356       if (!classValues.Contains(classValue)) throw new ArgumentException();
    357       int index = ClassValues.IndexOf(classValue);
    358       ClassNames[index] = className;
     357      if (!ClassValuesCache.Contains(classValue)) throw new ArgumentException();
     358      int index = ClassValuesCache.IndexOf(classValue);
    359359      ClassNamesParameter.Value[index, 0] = className;
     360      // updating of class names cache is not necessary here as the parameter value fires a changed event which updates the cache
    360361    }
    361362
    … …  
    364365    }
    365366    public double GetClassificationPenalty(double correctClassValue, double estimatedClassValue) {
    366       var key = Tuple.Create(correctClassValue, estimatedClassValue);
    367       if (!classificationPenaltiesCache.ContainsKey(key)) {
    368         int correctClassIndex = ClassValues.IndexOf(correctClassValue);
    369         int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
    370         classificationPenaltiesCache[key] = ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
    371       }
    372       return classificationPenaltiesCache[key];
     367      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
     368      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
     369      return ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex];
    373370    }
    374371    public void SetClassificationPenalty(string correctClassName, string estimatedClassName, double penalty) {
    … …  
    376373    }
    377374    public void SetClassificationPenalty(double correctClassValue, double estimatedClassValue, double penalty) {
    378       var key = Tuple.Create(correctClassValue, estimatedClassValue);
    379       int correctClassIndex = ClassValues.IndexOf(correctClassValue);
    380       int estimatedClassIndex = ClassValues.IndexOf(estimatedClassValue);
     375      int correctClassIndex = ClassValuesCache.IndexOf(correctClassValue);
     376      int estimatedClassIndex = ClassValuesCache.IndexOf(estimatedClassValue);
    381377
    382378      ClassificationPenaltiesParameter.Value[correctClassIndex, estimatedClassIndex] = penalty;
    … …  
    387383      TargetVariableParameter.ValueChanged += new EventHandler(TargetVariableParameter_ValueChanged);
    388384      ClassNamesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
    389       ClassNamesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     385      ClassNamesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
     386      ClassificationPenaltiesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
    390387      ClassificationPenaltiesParameter.Value.Reset += new EventHandler(Parameter_ValueChanged);
    391       ClassificationPenaltiesParameter.Value.ItemChanged += new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
    392388    }
    393389    private void DeregisterParameterEvents() {
    394390      TargetVariableParameter.ValueChanged -= new EventHandler(TargetVariableParameter_ValueChanged);
    395391      ClassNamesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
    396       ClassNamesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
     392      ClassNamesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
     393      ClassificationPenaltiesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(Parameter_ValueChanged);
    397394      ClassificationPenaltiesParameter.Value.Reset -= new EventHandler(Parameter_ValueChanged);
    398       ClassificationPenaltiesParameter.Value.ItemChanged -= new EventHandler<EventArgs<int, int>>(MatrixParameter_ItemChanged);
    399395    }
    400396
    401397    private void TargetVariableParameter_ValueChanged(object sender, EventArgs e) {
    402       classValues = null;
     398      classValuesCache = null;
     399      classNamesCache = null;
    403400      ResetTargetVariableDependentMembers();
    404401      OnChanged();
    405402    }
    406403    private void Parameter_ValueChanged(object sender, EventArgs e) {
    407       OnChanged();
    408     }
    409     private void MatrixParameter_ItemChanged(object sender, EventArgs<int, int> e) {
     404      classNamesCache = null;
     405      ClassificationPenaltiesParameter.Value.RowNames = ClassNames.Select(name => "Actual " + name);
     406      ClassificationPenaltiesParameter.Value.ColumnNames = ClassNames.Select(name => "Estimated " + name);
    410407      OnChanged();
    411408    }

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolution.cs

    r8508 r8811  
    4545      : base(model, problemData) {
    4646      evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
     47      CalculateClassificationResults();
    4748    }
    4849

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

    r8508 r8811  
    8585    }
    8686
    87     protected void CalculateResults() {
     87    protected void CalculateClassificationResults() {
    8888      double[] estimatedTrainingClassValues = EstimatedTrainingClassValues.ToArray(); // cache values
    8989      double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
    … …  
    114114
    115115    public abstract IEnumerable<double> GetEstimatedClassValues(IEnumerable<int> rows);
     116
     117    protected override void RecalculateResults() {
     118      CalculateClassificationResults();
     119    }
    116120  }
    117121}

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationModel.cs

    r7259 r8811  
    3333  [StorableClass]
    3434  [Item("DiscriminantFunctionClassificationModel", "Represents a classification model that uses a discriminant function and classification thresholds.")]
    35   public abstract class DiscriminantFunctionClassificationModel : NamedItem, IDiscriminantFunctionClassificationModel {
     35  public class DiscriminantFunctionClassificationModel : NamedItem, IDiscriminantFunctionClassificationModel {
    3636    [Storable]
    3737    private IRegressionModel model;
     38    public IRegressionModel Model {
     39      get { return model; }
     40      private set { model = value; }
     41    }
    3842
    3943    [Storable]
    … …  
    5155    }
    5256
     57    private IDiscriminantFunctionThresholdCalculator thresholdCalculator;
     58    [Storable]
     59    public IDiscriminantFunctionThresholdCalculator ThresholdCalculator {
     60      get { return thresholdCalculator; }
     61      private set { thresholdCalculator = value; }
     62    }
     63
    5364
    5465    [StorableConstructor]
    … …  
    6172    }
    6273
    63     public DiscriminantFunctionClassificationModel(IRegressionModel model)
     74    public DiscriminantFunctionClassificationModel(IRegressionModel model, IDiscriminantFunctionThresholdCalculator thresholdCalculator)
    6475      : base() {
    6576      this.name = ItemName;
    6677      this.description = ItemDescription;
    6778      this.model = model;
    68       this.classValues = new double[] { 0.0 };
    69       this.thresholds = new double[] { double.NegativeInfinity };
     79      this.classValues = new double[0];
     80      this.thresholds = new double[0];
     81      this.thresholdCalculator = thresholdCalculator;
     82    }
     83
     84    [StorableHook(HookType.AfterDeserialization)]
     85    private void AfterDeserialization() {
     86      if (ThresholdCalculator == null) ThresholdCalculator = new AccuracyMaximizationThresholdCalculator();
     87    }
     88
     89    public override IDeepCloneable Clone(Cloner cloner) {
     90      return new DiscriminantFunctionClassificationModel(this, cloner);
    7091    }
    7192
    … …  
    80101    }
    81102
     103    public virtual void RecalculateModelParameters(IClassificationProblemData problemData, IEnumerable<int> rows) {
     104      double[] classValues;
     105      double[] thresholds;
     106      var targetClassValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
     107      var estimatedTrainingValues = GetEstimatedValues(problemData.Dataset, rows);
     108      thresholdCalculator.Calculate(problemData, estimatedTrainingValues, targetClassValues, out classValues, out thresholds);
     109      SetThresholdsAndClassValues(thresholds, classValues);
     110    }
     111
     112
    82113    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
    83114      return model.GetEstimatedValues(dataset, rows);
    … …  
    85116
    86117    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
     118      if (!Thresholds.Any() && !ClassValues.Any()) throw new ArgumentException("No thresholds and class values were set for the current classification model.");
    87119      foreach (var x in GetEstimatedValues(dataset, rows)) {
    88120        int classIndex = 0;
    … …  
    103135    #endregion
    104136
    105     public abstract IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(IClassificationProblemData problemData);
    106     public abstract IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
     137    public virtual IDiscriminantFunctionClassificationSolution CreateDiscriminantFunctionClassificationSolution(IClassificationProblemData problemData) {
     138      return new DiscriminantFunctionClassificationSolution(this, problemData);
     139    }
     140
     141    public virtual IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
     142      return CreateDiscriminantFunctionClassificationSolution(problemData);
     143    }
    107144  }
    108145}

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolution.cs

    r8534 r8811  
    3232  [StorableClass]
    3333  [Item("DiscriminantFunctionClassificationSolution", "Represents a classification solution that uses a discriminant function and classification thresholds.")]
    34   public abstract class DiscriminantFunctionClassificationSolution : DiscriminantFunctionClassificationSolutionBase {
     34  public class DiscriminantFunctionClassificationSolution : DiscriminantFunctionClassificationSolutionBase {
    3535    protected readonly Dictionary<int, double> valueEvaluationCache;
    3636    protected readonly Dictionary<int, double> classValueEvaluationCache;
    … …  
    4747      classValueEvaluationCache = new Dictionary<int, double>(original.classValueEvaluationCache);
    4848    }
    49     protected DiscriminantFunctionClassificationSolution(IDiscriminantFunctionClassificationModel model, IClassificationProblemData problemData)
     49    public DiscriminantFunctionClassificationSolution(IDiscriminantFunctionClassificationModel model, IClassificationProblemData problemData)
    5050      : base(model, problemData) {
    5151      valueEvaluationCache = new Dictionary<int, double>();
    5252      classValueEvaluationCache = new Dictionary<int, double>();
     53      CalculateRegressionResults();
     54      CalculateClassificationResults();
     55    }
     56
     57    public override IDeepCloneable Clone(Cloner cloner) {
     58      return new DiscriminantFunctionClassificationSolution(this, cloner);
    5359    }
    5460

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolutionBase.cs

    r8534 r8811  
    8585      Add(new Result(TrainingRSquaredResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the training partition", new DoubleValue()));
    8686      Add(new Result(TestRSquaredResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the test partition", new DoubleValue()));
    87 
    8887      RegisterEventHandler();
    8988    }
    … …  
    9291    private void AfterDeserialization() {
    9392      RegisterEventHandler();
    94     }
    95 
    96     protected override void OnModelChanged() {
    97       DeregisterEventHandler();
    98       RegisterEventHandler();
    99       base.OnModelChanged();
    10093    }
    10194
    … …  
    137130
    138131    protected virtual void OnModelThresholdsChanged(EventArgs e) {
    139       CalculateResults();
    140       CalculateRegressionResults();
     132      OnModelChanged();
    141133    }
    142134
    … …  
    146138
    147139    public abstract IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows);
     140
     141    protected override void RecalculateResults() {
     142      base.RecalculateResults();
     143      CalculateRegressionResults();
     144    }
    148145  }
    149146}

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs

    r8508 r8811  
    5353
    5454    public static void CalculateThresholds(IClassificationProblemData problemData, IEnumerable<double> estimatedValues, IEnumerable<double> targetClassValues, out double[] classValues, out double[] thresholds) {
    55       int slices = 100;
    56       double minThresholdInc = 10e-5; // necessary to prevent infinite loop when maxEstimated - minEstimated is effectively zero (constant model)
     55      const int slices = 100;
     56      const double minThresholdInc = 10e-5; // necessary to prevent infinite loop when maxEstimated - minEstimated is effectively zero (constant model)
    5757      List<double> estimatedValuesList = estimatedValues.ToList();
    5858      double maxEstimatedValue = estimatedValuesList.Max();
    … …  
    6161      var estimatedAndTargetValuePairs =
    6262        estimatedValuesList.Zip(targetClassValues, (x, y) => new { EstimatedValue = x, TargetClassValue = y })
    63         .OrderBy(x => x.EstimatedValue)
    64         .ToList();
     63        .OrderBy(x => x.EstimatedValue).ToList();
    6564
    66       classValues = problemData.ClassValues.OrderBy(x => x).ToArray();
     65      classValues = estimatedAndTargetValuePairs.GroupBy(x => x.TargetClassValue)
     66        .Select(x => new { Median = x.Select(y => y.EstimatedValue).Median(), Class = x.Key })
     67        .OrderBy(x => x.Median).Select(x => x.Class).ToArray();
     68
    6769      int nClasses = classValues.Length;
    6870      thresholds = new double[nClasses];
    6971      thresholds[0] = double.NegativeInfinity;
    70       // thresholds[thresholds.Length - 1] = double.PositiveInfinity;
    7172
    7273      // incrementally calculate accuracy of all possible thresholds
    … …  
    8586            //all positives
    8687            if (pair.TargetClassValue.IsAlmost(classValues[i - 1])) {
    87               if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
     88              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
    8889                //true positive
    89                 classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i - 1]);
     90                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
    9091              else
    9192                //false negative
    92                 classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i - 1]);
     93                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i]);
    9394            }
    9495              //all negatives
    9596            else {
    96               if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue < actualThreshold)
    97                 //false positive
    98                 classificationScore += problemData.GetClassificationPenalty(classValues[i - 1], classValues[i]);
    99               else
    100                 //true negative, consider only upper class
    101                 classificationScore += problemData.GetClassificationPenalty(classValues[i], classValues[i]);
     97              //false positive
     98              if (pair.EstimatedValue > lowerThreshold && pair.EstimatedValue <= actualThreshold)
     99                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i - 1]);
     100              else if (pair.EstimatedValue <= lowerThreshold)
     101                classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i - 2]);
     102              else if (pair.EstimatedValue > actualThreshold) {
     103                if (pair.TargetClassValue < classValues[i - 1]) //negative in wrong class, consider upper class
     104                  classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, classValues[i]);
     105                else //true negative, must be optimized by the other thresholds
     106                  classificationScore += problemData.GetClassificationPenalty(pair.TargetClassValue, pair.TargetClassValue);
     107              }
    102108            }
    103109          }

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/NormalDistributionCutPointsThresholdCalculator.cs

    r7259 r8811  
    5353
    5454    public static void CalculateThresholds(IClassificationProblemData problemData, IEnumerable<double> estimatedValues, IEnumerable<double> targetClassValues, out double[] classValues, out double[] thresholds) {
    55       double maxEstimatedValue = estimatedValues.Max();
    56       double minEstimatedValue = estimatedValues.Min();
    5755      var estimatedTargetValues = Enumerable.Zip(estimatedValues, targetClassValues, (e, t) => new { EstimatedValue = e, TargetValue = t }).ToList();
     56      double estimatedValuesRange = estimatedValues.Range();
    5857
    5958      Dictionary<double, double> classMean = new Dictionary<double, double>();
    … …  
    8281          // calculate all thresholds
    8382          CalculateCutPoints(classMean[class0], classStdDev[class0], classMean[class1], classStdDev[class1], out x1, out x2);
    84           if (!thresholdList.Any(x => x.IsAlmost(x1))) thresholdList.Add(x1);
    85           if (!thresholdList.Any(x => x.IsAlmost(x2))) thresholdList.Add(x2);
     83
     84          // if the two cut points are too close (for instance because the stdDev=0)
     85          // then move them by 0.1% of the range of estimated values
     86          if (x1.IsAlmost(x2)) {
     87            x1 -= 0.001 * estimatedValuesRange;
     88            x2 += 0.001 * estimatedValuesRange;
     89          }
     90          if (!double.IsInfinity(x1) && !thresholdList.Any(x => x.IsAlmost(x1))) thresholdList.Add(x1);
     91          if (!double.IsInfinity(x2) && !thresholdList.Any(x => x.IsAlmost(x2))) thresholdList.Add(x2);
    8692        }
    8793      }
    8894      thresholdList.Sort();
    89       thresholdList.Insert(0, double.NegativeInfinity);
     95
     96      // add small value and large value for the calculation of most influential class in each thresholded section
     97      thresholdList.Insert(0, estimatedValues.Min() - 1);
     98      thresholdList.Add(estimatedValues.Max() + 1);
    9099
    91100      // determine class values for each partition separated by a threshold by calculating the density of all class distributions
    92101      // all points in the partition are classified as the class with the maximal density in the parition
    93102      List<double> classValuesList = new List<double>();
    94       for (int i = 0; i < thresholdList.Count; i++) {
    95         double m;
    96         if (double.IsNegativeInfinity(thresholdList[i])) {
    97           m = thresholdList[i + 1] - 1.0; // smaller than the smalles non-infinity threshold
    98         } else if (i == thresholdList.Count - 1) {
    99           // last threshold
    100           m = thresholdList[i] + 1.0; // larger than the last threshold
    101         } else {
    102           m = thresholdList[i] + (thresholdList[i + 1] - thresholdList[i]) / 2.0; // middle of partition
     103      if (thresholdList.Count == 2) {
     104        // this happens if there are no thresholds (distributions for all classes are exactly the same)
     105        // -> all samples should be classified as the first class
     106        classValuesList.Add(originalClasses[0]);
     107      } else {
     108        // at least one reasonable threshold ...
     109        // find the most likely class for the points between thresholds m
     110        for (int i = 0; i < thresholdList.Count - 1; i++) {
     111
     112          // determine class with maximal density mass between the thresholds
     113          double maxDensity = LogNormalDensityMass(thresholdList[i], thresholdList[i + 1], classMean[originalClasses[0]], classStdDev[originalClasses[0]]);
     114          double maxDensityClassValue = originalClasses[0];
     115          foreach (var classValue in originalClasses.Skip(1)) {
     116            double density = LogNormalDensityMass(thresholdList[i], thresholdList[i + 1], classMean[classValue], classStdDev[classValue]);
     117            if (density > maxDensity) {
     118              maxDensity = density;
     119              maxDensityClassValue = classValue;
     120            }
     121          }
     122          classValuesList.Add(maxDensityClassValue);
    103123        }
    104 
    105         // determine class with maximal probability density in m
    106         double maxDensity = double.MinValue;
    107         double maxDensityClassValue = -1;
    108         foreach (var classValue in originalClasses) {
    109           double density = NormalDensity(m, classMean[classValue], classStdDev[classValue]);
    110           if (density > maxDensity) {
    111             maxDensity = density;
    112             maxDensityClassValue = classValue;
    113           }
    114         }
    115         classValuesList.Add(maxDensityClassValue);
    116124      }
    117125
    … …  
    125133      //    /   / /\s  \ \     
    126134      //  -/---/-/ -\---\-\----
     135
    127136      List<double> filteredThresholds = new List<double>();
    128137      List<double> filteredClassValues = new List<double>();
    129       filteredThresholds.Add(thresholdList[0]);
     138      filteredThresholds.Add(double.NegativeInfinity); // the smallest possible threshold for the first class
    130139      filteredClassValues.Add(classValuesList[0]);
     140      // do not include the last threshold which was just needed for the previous step
    131141      for (int i = 0; i < classValuesList.Count - 1; i++) {
    132         if (classValuesList[i] != classValuesList[i + 1]) {
     142        if (!classValuesList[i].IsAlmost(classValuesList[i + 1])) {
    133143          filteredThresholds.Add(thresholdList[i + 1]);
    134144          filteredClassValues.Add(classValuesList[i + 1]);
    … …  
    139149    }
    140150
    141     private static double NormalDensity(double x, double mu, double sigma) {
     151    private static double LogNormalDensityMass(double lower, double upper, double mu, double sigma) {
    142152      if (sigma.IsAlmost(0.0)) {
    143         if (x.IsAlmost(mu)) return 1.0; else return 0.0;
    144       } else {
    145         return (1.0 / Math.Sqrt(2.0 * Math.PI * sigma * sigma)) * Math.Exp(-((x - mu) * (x - mu)) / (2.0 * sigma * sigma));
     153        if (lower < mu && mu < upper) return double.PositiveInfinity; // log(1)
     154        else return double.NegativeInfinity; // log(0)
    146155      }
     156
     157      Func<double, double> f = (x) =>
     158        x * -0.5 * Math.Log(2.0 * Math.PI * sigma * sigma) - Math.Pow(x - mu, 3) / (3 * 2.0 * sigma * sigma);
     159
     160      if (double.IsNegativeInfinity(lower)) return f(upper);
     161      else return f(upper) - f(lower);
    147162    }
    148163
    149164    private static void CalculateCutPoints(double m1, double s1, double m2, double s2, out double x1, out double x2) {
    150       double a = (s1 * s1 - s2 * s2);
    151       x1 = -(-m2 * s1 * s1 + m1 * s2 * s2 + Math.Sqrt(s1 * s1 * s2 * s2 * ((m1 - m2) * (m1 - m2) + 2.0 * (-s1 * s1 + s2 * s2) * Math.Log(s2 / s1)))) / a;
    152       x2 = (m2 * s1 * s1 - m1 * s2 * s2 + Math.Sqrt(s1 * s1 * s2 * s2 * ((m1 - m2) * (m1 - m2) + 2.0 * (-s1 * s1 + s2 * s2) * Math.Log(s2 / s1)))) / a;
     165      if (s1.IsAlmost(s2)) {
     166        if (m1.IsAlmost(m2)) {
     167          x1 = double.NegativeInfinity;
     168          x2 = double.NegativeInfinity;
     169        } else {
     170          x1 = (m1 + m2) / 2;
     171          x2 = double.NegativeInfinity;
     172        }
     173      } else if (s1.IsAlmost(0.0)) {
     174        x1 = m1;
     175        x2 = m1;
     176      } else if (s2.IsAlmost(0.0)) {
     177        x1 = m2;
     178        x2 = m2;
     179      } else {
     180        if (s2 < s1) {
     181          // make sure s2 is the larger std.dev.
     182          CalculateCutPoints(m2, s2, m1, s1, out x1, out x2);
     183        } else {
     184          double a = (s1 + s2) * (s1 - s2);
     185          double g = Math.Sqrt(s1 * s1 * s2 * s2 * ((m1 - m2) * (m1 - m2) + 2.0 * (s1 * s1 + s2 * s2) * Math.Log(s2 / s1)));
     186          double m1s2 = m1 * s2 * s2;
     187          double m2s1 = m2 * s1 * s1;
     188          x1 = -(-m2s1 + m1s2 + g) / a;
     189          x2 = (m2s1 - m1s2 + g) / a;
     190        }
     191      }
    153192    }
    154193  }

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/ClassificationWeightCalculator.cs

    r8297 r8811  
    2626using HeuristicLab.Data;
    2727using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    28 using HeuristicLab.Problems.DataAnalysis.Interfaces;
    2928
    3029namespace HeuristicLab.Problems.DataAnalysis {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/WeightCalculators/DiscriminantClassificationWeightCalculator.cs

    r8297 r8811  
    2424using HeuristicLab.Common;
    2525using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    26 using HeuristicLab.Problems.DataAnalysis.Interfaces;
    2726
    2827namespace HeuristicLab.Problems.DataAnalysis {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblem.cs

    r7866 r8811  
    6666    }
    6767
     68    protected DataAnalysisProblem(T problemData)
     69      : this() {
     70      ProblemData = problemData;
     71    }
     72
    6873    [StorableHook(HookType.AfterDeserialization)]
    6974    private void AfterDeserialization() {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblemData.cs

    r8508 r8811  
    107107    [StorableConstructor]
    108108    protected DataAnalysisProblemData(bool deserializing) : base(deserializing) { }
     109
    109110    [StorableHook(HookType.AfterDeserialization)]
    110111    private void AfterDeserialization() {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/ConstantRegressionModel.cs

    r8534 r8811  
    3030  public class ConstantRegressionModel : NamedItem, IRegressionModel {
    3131    [Storable]
    32     private double constant;
     32    protected double constant;
    3333    public double Constant {
    3434      get { return constant; }

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionEnsembleSolution.cs

    r8508 r8811  
    3636  [Item("Regression Ensemble Solution", "A regression solution that contains an ensemble of multiple regression models")]
    3737  [Creatable("Data Analysis - Ensembles")]
    38   public sealed class RegressionEnsembleSolution : RegressionSolution, IRegressionEnsembleSolution {
     38  public sealed class RegressionEnsembleSolution : RegressionSolutionBase, IRegressionEnsembleSolution {
    3939    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
    4040    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
     41    private readonly Dictionary<int, double> evaluationCache = new Dictionary<int, double>();
    4142
    4243    public new IRegressionEnsembleModel Model {
    … …  
    155156    }
    156157
    157     protected override void RecalculateResults() {
    158       CalculateResults();
    159     }
    160 
    161158    #region Evaluation
     159    public override IEnumerable<double> EstimatedValues {
     160      get { return GetEstimatedValues(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
     161    }
     162
    162163    public override IEnumerable<double> EstimatedTrainingValues {
    163164      get {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblem.cs

    r7866 r8811  
    3636    public override IDeepCloneable Clone(Cloner cloner) { return new RegressionProblem(this, cloner); }
    3737
    38     public RegressionProblem()
    39       : base() {
    40       ProblemData = new RegressionProblemData();
    41     }
     38    public RegressionProblem() : base(new RegressionProblemData()) { }
     39
    4240  }
    4341}

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolution.cs

    r8508 r8811  
    4545      : base(model, problemData) {
    4646      evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
     47      CalculateRegressionResults();
    4748    }
    4849
    49     protected override void RecalculateResults() {
    50       CalculateResults();
    51     }
    5250
    5351    public override IEnumerable<double> EstimatedValues {

  • branches/ClassificationEnsembleVoting/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionBase.cs

    r8508 r8811  
    2929  [StorableClass]
    3030  public abstract class RegressionSolutionBase : DataAnalysisSolution, IRegressionSolution {
    31     private const string TrainingMeanSquaredErrorResultName = "Mean squared error (training)";
    32     private const string TestMeanSquaredErrorResultName = "Mean squared error (test)";
    33     private const string TrainingMeanAbsoluteErrorResultName = "Mean absolute error (training)";
    34     private const string TestMeanAbsoluteErrorResultName = "Mean absolute error (test)";
    35     private const string TrainingSquaredCorrelationResultName = "Pearson's R² (training)";
    36     private const string TestSquaredCorrelationResultName = "Pearson's R² (test)";
    37     private const string TrainingRelativeErrorResultName = "Average relative error (training)";
    38     private const string TestRelativeErrorResultName = "Average relative error (test)";
    39     private const string TrainingNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (training)";
    40     private const string TestNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (test)";
    41     private const string TrainingMeanErrorResultName = "Mean error (training)";
    42     private const string TestMeanErrorResultName = "Mean error (test)";
     31    protected const string TrainingMeanSquaredErrorResultName = "Mean squared error (training)";
     32    protected const string TestMeanSquaredErrorResultName = "Mean squared error (test)";
     33    protected const string TrainingMeanAbsoluteErrorResultName = "Mean absolute error (training)";
     34    protected const string TestMeanAbsoluteErrorResultName = "Mean absolute error (test)";
     35    protected const string TrainingSquaredCorrelationResultName = "Pearson's R² (training)";
     36    protected const string TestSquaredCorrelationResultName = "Pearson's R² (test)";
     37    protected const string TrainingRelativeErrorResultName = "Average relative error (training)";
     38    protected const string TestRelativeErrorResultName = "Average relative error (test)";
     39    protected const string TrainingNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (training)";
     40    protected const string TestNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (test)";
     41    protected const string TrainingMeanErrorResultName = "Mean error (training)";
     42    protected const string TestMeanErrorResultName = "Mean error (test)";
     43
     44    protected const string TrainingMeanSquaredErrorResultDescription = "Mean of squared errors of the model on the training partition";
     45    protected const string TestMeanSquaredErrorResultDescription = "Mean of squared errors of the model on the test partition";
     46    protected const string TrainingMeanAbsoluteErrorResultDescription = "Mean of absolute errors of the model on the training partition";
     47    protected const string TestMeanAbsoluteErrorResultDescription = "Mean of absolute errors of the model on the test partition";
     48    protected const string TrainingSquaredCorrelationResultDescription = "Squared Pearson's correlation coefficient of the model output and the actual values on the training partition";
     49    protected const string TestSquaredCorrelationResultDescription = "Squared Pearson's correlation coefficient of the model output and the actual values on the test partition";
     50    protected const string TrainingRelativeErrorResultDescription = "Average of the relative errors of the model output and the actual values on the training partition";
     51    protected const string TestRelativeErrorResultDescription = "Average of the relative errors of the model output and the actual values on the test partition";
     52    protected const string TrainingNormalizedMeanSquaredErrorResultDescription = "Normalized mean of squared errors of the model on the training partition";
     53    protected const string TestNormalizedMeanSquaredErrorResultDescription = "Normalized mean of squared errors of the model on the test partition";
     54    protected const string TrainingMeanErrorResultDescription = "Mean of errors of the model on the training partition";
     55    protected const string TestMeanErrorResultDescription = "Mean of errors of the model on the test partition";
    4356
    4457    public new IRegressionModel Model {
    … …  
    115128    protected RegressionSolutionBase(IRegressionModel model, IRegressionProblemData problemData)
    116129      : base(model, problemData) {
    117       Add(new Result(TrainingMeanSquaredErrorResultName, "Mean of squared errors of the model on the training partition", new DoubleValue()));
    118       Add(new Result(TestMeanSquaredErrorResultName, "Mean of squared errors of the model on the test partition", new DoubleValue()));
    119       Add(new Result(TrainingMeanAbsoluteErrorResultName, "Mean of absolute errors of the model on the training partition", new DoubleValue()));
    120       Add(new Result(TestMeanAbsoluteErrorResultName, "Mean of absolute errors of the model on the test partition", new DoubleValue()));
    121       Add(new Result(TrainingSquaredCorrelationResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the training partition", new DoubleValue()));
    122       Add(new Result(TestSquaredCorrelationResultName, "Squared Pearson's correlation coefficient of the model output and the actual values on the test partition", new DoubleValue()));
    123       Add(new Result(TrainingRelativeErrorResultName, "Average of the relative errors of the model output and the actual values on the training partition", new PercentValue()));
    124       Add(new Result(TestRelativeErrorResultName, "Average of the relative errors of the model output and the actual values on the test partition", new PercentValue()));
    125       Add(new Result(TrainingNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the training partition", new DoubleValue()));
    126       Add(new Result(TestNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the test partition", new DoubleValue()));
    127       Add(new Result(TrainingMeanErrorResultName, "Mean of errors of the model on the training partition", new DoubleValue()));
    128       Add(new Result(TestMeanErrorResultName, "Mean of errors of the model on the test partition", new DoubleValue()));
     130      Add(new Result(TrainingMeanSquaredErrorResultName, TrainingMeanSquaredErrorResultDescription, new DoubleValue()));
     131      Add(new Result(TestMeanSquaredErrorResultName, TestMeanSquaredErrorResultDescription, new DoubleValue()));
     132      Add(new Result(TrainingMeanAbsoluteErrorResultName, TrainingMeanAbsoluteErrorResultDescription, new DoubleValue()));
     133      Add(new Result(TestMeanAbsoluteErrorResultName, TestMeanAbsoluteErrorResultDescription, new DoubleValue()));
     134      Add(new Result(TrainingSquaredCorrelationResultName, TrainingSquaredCorrelationResultDescription, new DoubleValue()));
     135      Add(new Result(TestSquaredCorrelationResultName, TestSquaredCorrelationResultDescription, new DoubleValue()));
     136      Add(new Result(TrainingRelativeErrorResultName, TrainingRelativeErrorResultDescription, new PercentValue()));
     137      Add(new Result(TestRelativeErrorResultName, TestRelativeErrorResultDescription, new PercentValue()));
     138      Add(new Result(TrainingNormalizedMeanSquaredErrorResultName, TrainingNormalizedMeanSquaredErrorResultDescription, new DoubleValue()));
     139      Add(new Result(TestNormalizedMeanSquaredErrorResultName, TestNormalizedMeanSquaredErrorResultDescription, new DoubleValue()));
     140      Add(new Result(TrainingMeanErrorResultName, TrainingMeanErrorResultDescription, new DoubleValue()));
     141      Add(new Result(TestMeanErrorResultName, TestMeanErrorResultDescription, new DoubleValue()));
    129142    }
    130143
    … …  
    164177    }
    165178
    166     protected void CalculateResults() {
     179    protected override void RecalculateResults() {
     180      CalculateRegressionResults();
     181    }
     182
     183    protected void CalculateRegressionResults() {
    167184      IEnumerable<double> estimatedTrainingValues = EstimatedTrainingValues; // cache values
    168185      IEnumerable<double> originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
Note: See TracChangeset for help on using the changeset viewer.