Changeset 13797 for branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

Timestamp:

04/26/16 13:07:26 (8 years ago)

Author:

abeham

Message:

#2457: worked on testing recommendation algorithms through x-validation

File:

• branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs (modified) (3 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

@@ -31 +31 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace HeuristicLab.OptimizationExpertSystem {
@@ -131 +132 @@
     private void xValidateButton_Click(object sender, EventArgs e) {
       var recommender = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
+      var progress = MainForm.AddOperationProgressToView(this, "Performing Leave-one-out Crossvalidation");
+
+      Task.Factory.StartNew(() => { DoCrossvalidate(recommender, progress); }, TaskCreationOptions.LongRunning);
+    }
+
+    private void DoCrossvalidate(IAlgorithmInstanceRecommender recommender, IProgress progress) {
       var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();
-
       var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
 
       var absErr = 0.0;
+      var absLogError = 0.0;
       var confMatrix = new int[6, 6];
       var tau = 0.0;
       // leave one out crossvalidation
+      var count = 0;
       foreach (var pi in trainingSet) {
+        progress.Status = pi.Name + "...";
         var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
         var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
         var observed = Content.GetAlgorithmPerformance(pi);
         absErr += AbsoluteError(observed, predicted);
+        absLogError += AbsoluteLogError(observed, predicted);
         var confMat = ConfusionMatrix(observed, predicted);
-        for (var i = 0; i < confMat.GetLength(0); i++)
+        for (var i = 0; i < confMat.GetLength(0); i++) {
           for (var j = 0; j < confMat.GetLength(1); j++)
             confMatrix[i, j] += confMat[i, j];
+        }
         tau += KendallsTau(observed, predicted);
         // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
         // mean reciprocal rank
         // optional: expected reciprocal rank
+        progress.ProgressValue = ++count / (double)trainingSet.Length;
       }
       absErr /= trainingSet.Length;
+      absLogError /= trainingSet.Length;
       tau /= trainingSet.Length;
 
-      confusionMatrixView.Content = new IntMatrix(confMatrix);
       absoluteErrorView.Content = new DoubleValue(absErr);
+      absoluteLogErrorView.Content = new DoubleValue(absLogError);
+      var description = new[] { "A", "B", "C", "D", "E", "F" };
+      confusionMatrixView.Content = new IntMatrix(confMatrix) { ColumnNames = description, RowNames = description };
       kendallsTauView.Content = new DoubleValue(tau);
+
+      progress.Finish();
     }
 
@@ -168 +185 @@
         if (double.IsNaN(actual)) actual = int.MaxValue;
         error += Math.Abs(actual - tuple.Value);
+      }
+      return error;
+    }
+
+    private static double AbsoluteLogError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
+      var error = 0.0;
+      foreach (var tuple in ranking) {
+        double actual;
+        if (!performance.TryGetValue(tuple.Key, out actual)) continue;
+        if (double.IsNaN(actual)) actual = int.MaxValue;
+        error += Math.Abs(Math.Log10(actual) - Math.Log10(tuple.Value));
       }
       return error;