Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
04/26/16 13:07:26 (9 years ago)
Author:
abeham
Message:

#2457: worked on testing recommendation algorithms through x-validation

Location:
branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs

    r13794 r13797  
    5656      this.crossvalidationTabPage = new System.Windows.Forms.TabPage();
    5757      this.kendallsTauLabel = new System.Windows.Forms.Label();
     58      this.predictedLabel = new System.Windows.Forms.Label();
     59      this.actualLabel = new System.Windows.Forms.Label();
     60      this.absoluteLogErrorLabel = new System.Windows.Forms.Label();
    5861      this.absoluteErrorLabel = new System.Windows.Forms.Label();
    5962      this.kendallsTauView = new HeuristicLab.Data.Views.StringConvertibleValueView();
     63      this.absoluteLogErrorView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6064      this.absoluteErrorView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6165      this.confusionMatrixView = new HeuristicLab.Data.Views.StringConvertibleMatrixView();
    … …  
    6367      this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6468      this.minimumTargetLabel = new System.Windows.Forms.Label();
    65       this.actualLabel = new System.Windows.Forms.Label();
    66       this.predictedLabel = new System.Windows.Forms.Label();
    6769      this.tabControl.SuspendLayout();
    6870      this.characteristicsTabPage.SuspendLayout();
    … …  
    111113      this.tabControl.Name = "tabControl";
    112114      this.tabControl.SelectedIndex = 0;
    113       this.tabControl.Size = new System.Drawing.Size(817, 509);
     115      this.tabControl.Size = new System.Drawing.Size(960, 509);
    114116      this.tabControl.TabIndex = 14;
    115117      //
    … …  
    121123      this.characteristicsTabPage.Name = "characteristicsTabPage";
    122124      this.characteristicsTabPage.Padding = new System.Windows.Forms.Padding(3);
    123       this.characteristicsTabPage.Size = new System.Drawing.Size(692, 324);
     125      this.characteristicsTabPage.Size = new System.Drawing.Size(952, 483);
    124126      this.characteristicsTabPage.TabIndex = 2;
    125127      this.characteristicsTabPage.Text = "Characteristics";
    … …  
    158160      this.parametersTabPage.Name = "parametersTabPage";
    159161      this.parametersTabPage.Padding = new System.Windows.Forms.Padding(3);
    160       this.parametersTabPage.Size = new System.Drawing.Size(692, 324);
     162      this.parametersTabPage.Size = new System.Drawing.Size(952, 483);
    161163      this.parametersTabPage.TabIndex = 0;
    162164      this.parametersTabPage.Text = "Parameters";
    … …  
    173175      this.parameterCollectionView.ReadOnly = false;
    174176      this.parameterCollectionView.ShowDetails = true;
    175       this.parameterCollectionView.Size = new System.Drawing.Size(686, 318);
     177      this.parameterCollectionView.Size = new System.Drawing.Size(946, 477);
    176178      this.parameterCollectionView.TabIndex = 0;
    177179      //
    … …  
    181183      this.crossvalidationTabPage.Controls.Add(this.predictedLabel);
    182184      this.crossvalidationTabPage.Controls.Add(this.actualLabel);
     185      this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorLabel);
    183186      this.crossvalidationTabPage.Controls.Add(this.absoluteErrorLabel);
    184187      this.crossvalidationTabPage.Controls.Add(this.kendallsTauView);
     188      this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorView);
    185189      this.crossvalidationTabPage.Controls.Add(this.absoluteErrorView);
    186190      this.crossvalidationTabPage.Controls.Add(this.confusionMatrixView);
    … …  
    189193      this.crossvalidationTabPage.Name = "crossvalidationTabPage";
    190194      this.crossvalidationTabPage.Padding = new System.Windows.Forms.Padding(3);
    191       this.crossvalidationTabPage.Size = new System.Drawing.Size(809, 483);
     195      this.crossvalidationTabPage.Size = new System.Drawing.Size(952, 483);
    192196      this.crossvalidationTabPage.TabIndex = 1;
    193197      this.crossvalidationTabPage.Text = "Crossvalidation";
    … …  
    197201      //
    198202      this.kendallsTauLabel.AutoSize = true;
    199       this.kendallsTauLabel.Location = new System.Drawing.Point(536, 110);
     203      this.kendallsTauLabel.Location = new System.Drawing.Point(519, 165);
    200204      this.kendallsTauLabel.Margin = new System.Windows.Forms.Padding(3);
    201205      this.kendallsTauLabel.Name = "kendallsTauLabel";
    … …  
    204208      this.kendallsTauLabel.Text = "Mean Kendall\'s Tau:";
    205209      //
     210      // predictedLabel
     211      //
     212      this.predictedLabel.AutoSize = true;
     213      this.predictedLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
     214      this.predictedLabel.Location = new System.Drawing.Point(223, 52);
     215      this.predictedLabel.Margin = new System.Windows.Forms.Padding(3);
     216      this.predictedLabel.Name = "predictedLabel";
     217      this.predictedLabel.Size = new System.Drawing.Size(69, 13);
     218      this.predictedLabel.TabIndex = 3;
     219      this.predictedLabel.Text = "PREDICTED";
     220      //
     221      // actualLabel
     222      //
     223      this.actualLabel.AutoSize = true;
     224      this.actualLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
     225      this.actualLabel.Location = new System.Drawing.Point(16, 155);
     226      this.actualLabel.Margin = new System.Windows.Forms.Padding(3);
     227      this.actualLabel.Name = "actualLabel";
     228      this.actualLabel.Size = new System.Drawing.Size(15, 78);
     229      this.actualLabel.TabIndex = 3;
     230      this.actualLabel.Text = "A\r\nC\r\nT\r\nU\r\nA\r\nL";
     231      //
     232      // absoluteLogErrorLabel
     233      //
     234      this.absoluteLogErrorLabel.AutoSize = true;
     235      this.absoluteLogErrorLabel.Location = new System.Drawing.Point(519, 109);
     236      this.absoluteLogErrorLabel.Margin = new System.Windows.Forms.Padding(3);
     237      this.absoluteLogErrorLabel.Name = "absoluteLogErrorLabel";
     238      this.absoluteLogErrorLabel.Size = new System.Drawing.Size(127, 13);
     239      this.absoluteLogErrorLabel.TabIndex = 3;
     240      this.absoluteLogErrorLabel.Text = "Mean Absolute Log Error:";
     241      //
    206242      // absoluteErrorLabel
    207243      //
    208244      this.absoluteErrorLabel.AutoSize = true;
    209       this.absoluteErrorLabel.Location = new System.Drawing.Point(536, 52);
     245      this.absoluteErrorLabel.Location = new System.Drawing.Point(519, 52);
    210246      this.absoluteErrorLabel.Margin = new System.Windows.Forms.Padding(3);
    211247      this.absoluteErrorLabel.Name = "absoluteErrorLabel";
    … …  
    219255      this.kendallsTauView.Content = null;
    220256      this.kendallsTauView.LabelVisible = false;
    221       this.kendallsTauView.Location = new System.Drawing.Point(520, 129);
     257      this.kendallsTauView.Location = new System.Drawing.Point(503, 184);
    222258      this.kendallsTauView.Name = "kendallsTauView";
    223259      this.kendallsTauView.ReadOnly = true;
    … …  
    225261      this.kendallsTauView.TabIndex = 2;
    226262      //
     263      // absoluteLogErrorView
     264      //
     265      this.absoluteLogErrorView.Caption = "StringConvertibleValue View";
     266      this.absoluteLogErrorView.Content = null;
     267      this.absoluteLogErrorView.LabelVisible = false;
     268      this.absoluteLogErrorView.Location = new System.Drawing.Point(503, 128);
     269      this.absoluteLogErrorView.Name = "absoluteLogErrorView";
     270      this.absoluteLogErrorView.ReadOnly = true;
     271      this.absoluteLogErrorView.Size = new System.Drawing.Size(194, 21);
     272      this.absoluteLogErrorView.TabIndex = 2;
     273      //
    227274      // absoluteErrorView
    228275      //
    … …  
    230277      this.absoluteErrorView.Content = null;
    231278      this.absoluteErrorView.LabelVisible = false;
    232       this.absoluteErrorView.Location = new System.Drawing.Point(520, 71);
     279      this.absoluteErrorView.Location = new System.Drawing.Point(503, 71);
    233280      this.absoluteErrorView.Name = "absoluteErrorView";
    234281      this.absoluteErrorView.ReadOnly = true;
    … …  
    245292      this.confusionMatrixView.ShowRowsAndColumnsTextBox = false;
    246293      this.confusionMatrixView.ShowStatisticalInformation = false;
    247       this.confusionMatrixView.Size = new System.Drawing.Size(477, 363);
     294      this.confusionMatrixView.Size = new System.Drawing.Size(438, 244);
    248295      this.confusionMatrixView.TabIndex = 1;
    249296      //
    … …  
    277324      this.minimumTargetLabel.TabIndex = 17;
    278325      this.minimumTargetLabel.Text = "Target:";
    279       //
    280       // actualLabel
    281       //
    282       this.actualLabel.AutoSize = true;
    283       this.actualLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
    284       this.actualLabel.Location = new System.Drawing.Point(16, 210);
    285       this.actualLabel.Margin = new System.Windows.Forms.Padding(3);
    286       this.actualLabel.Name = "actualLabel";
    287       this.actualLabel.Size = new System.Drawing.Size(15, 78);
    288       this.actualLabel.TabIndex = 3;
    289       this.actualLabel.Text = "A\r\nC\r\nT\r\nU\r\nA\r\nL";
    290       //
    291       // predictedLabel
    292       //
    293       this.predictedLabel.AutoSize = true;
    294       this.predictedLabel.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
    295       this.predictedLabel.Location = new System.Drawing.Point(241, 52);
    296       this.predictedLabel.Margin = new System.Windows.Forms.Padding(3);
    297       this.predictedLabel.Name = "predictedLabel";
    298       this.predictedLabel.Size = new System.Drawing.Size(69, 13);
    299       this.predictedLabel.TabIndex = 3;
    300       this.predictedLabel.Text = "PREDICTED";
    301326      //
    302327      // PerformanceModelingView
    … …  
    311336      this.Controls.Add(this.recommendStartButton);
    312337      this.Name = "PerformanceModelingView";
    313       this.Size = new System.Drawing.Size(817, 539);
     338      this.Size = new System.Drawing.Size(960, 539);
    314339      this.tabControl.ResumeLayout(false);
    315340      this.characteristicsTabPage.ResumeLayout(false);
    … …  
    344369    private System.Windows.Forms.Label predictedLabel;
    345370    private System.Windows.Forms.Label actualLabel;
     371    private System.Windows.Forms.Label absoluteLogErrorLabel;
     372    private Data.Views.StringConvertibleValueView absoluteLogErrorView;
    346373  }
    347374}

  • branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

    r13794 r13797  
    3131using System.Collections.Generic;
    3232using System.Linq;
     33using System.Threading.Tasks;
    3334
    3435namespace HeuristicLab.OptimizationExpertSystem {
    … …  
    131132    private void xValidateButton_Click(object sender, EventArgs e) {
    132133      var recommender = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
     134      var progress = MainForm.AddOperationProgressToView(this, "Performing Leave-one-out Crossvalidation");
     135
     136      Task.Factory.StartNew(() => { DoCrossvalidate(recommender, progress); }, TaskCreationOptions.LongRunning);
     137    }
     138
     139    private void DoCrossvalidate(IAlgorithmInstanceRecommender recommender, IProgress progress) {
    133140      var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();
    134 
    135141      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
    136142
    137143      var absErr = 0.0;
     144      var absLogError = 0.0;
    138145      var confMatrix = new int[6, 6];
    139146      var tau = 0.0;
    140147      // leave one out crossvalidation
     148      var count = 0;
    141149      foreach (var pi in trainingSet) {
     150        progress.Status = pi.Name + "...";
    142151        var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
    143152        var predicted = model.GetRanking(pi).ToDictionary(x => x.Key, x => x.Value);
    144153        var observed = Content.GetAlgorithmPerformance(pi);
    145154        absErr += AbsoluteError(observed, predicted);
     155        absLogError += AbsoluteLogError(observed, predicted);
    146156        var confMat = ConfusionMatrix(observed, predicted);
    147         for (var i = 0; i < confMat.GetLength(0); i++)
     157        for (var i = 0; i < confMat.GetLength(0); i++) {
    148158          for (var j = 0; j < confMat.GetLength(1); j++)
    149159            confMatrix[i, j] += confMat[i, j];
     160        }
    150161        tau += KendallsTau(observed, predicted);
    151162        // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
    152163        // mean reciprocal rank
    153164        // optional: expected reciprocal rank
     165        progress.ProgressValue = ++count / (double)trainingSet.Length;
    154166      }
    155167      absErr /= trainingSet.Length;
     168      absLogError /= trainingSet.Length;
    156169      tau /= trainingSet.Length;
    157170
    158       confusionMatrixView.Content = new IntMatrix(confMatrix);
    159171      absoluteErrorView.Content = new DoubleValue(absErr);
     172      absoluteLogErrorView.Content = new DoubleValue(absLogError);
     173      var description = new[] { "A", "B", "C", "D", "E", "F" };
     174      confusionMatrixView.Content = new IntMatrix(confMatrix) { ColumnNames = description, RowNames = description };
    160175      kendallsTauView.Content = new DoubleValue(tau);
     176
     177      progress.Finish();
    161178    }
    162179
    … …  
    168185        if (double.IsNaN(actual)) actual = int.MaxValue;
    169186        error += Math.Abs(actual - tuple.Value);
     187      }
     188      return error;
     189    }
     190
     191    private static double AbsoluteLogError(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
     192      var error = 0.0;
     193      foreach (var tuple in ranking) {
     194        double actual;
     195        if (!performance.TryGetValue(tuple.Key, out actual)) continue;
     196        if (double.IsNaN(actual)) actual = int.MaxValue;
     197        error += Math.Abs(Math.Log10(actual) - Math.Log10(tuple.Value));
    170198      }
    171199      return error;

  • branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/UnderstandingProblemInstanceView.Designer.cs

    r13791 r13797  
    147147      this.invPropCheckBox.Checked = true;
    148148      this.invPropCheckBox.CheckState = System.Windows.Forms.CheckState.Checked;
    149       this.invPropCheckBox.Location = new System.Drawing.Point(515, 8);
     149      this.invPropCheckBox.Location = new System.Drawing.Point(577, 8);
    150150      this.invPropCheckBox.Name = "invPropCheckBox";
    151151      this.invPropCheckBox.Size = new System.Drawing.Size(118, 17);
    … …  
    158158      //
    159159      this.colorLabel.AutoSize = true;
    160       this.colorLabel.Location = new System.Drawing.Point(652, 9);
     160      this.colorLabel.Location = new System.Drawing.Point(714, 9);
    161161      this.colorLabel.Name = "colorLabel";
    162162      this.colorLabel.Size = new System.Drawing.Size(34, 13);
    … …  
    222222      this.colorComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
    223223      this.colorComboBox.FormattingEnabled = true;
    224       this.colorComboBox.Location = new System.Drawing.Point(692, 6);
     224      this.colorComboBox.Location = new System.Drawing.Point(754, 6);
    225225      this.colorComboBox.Name = "colorComboBox";
    226       this.colorComboBox.Size = new System.Drawing.Size(222, 21);
     226      this.colorComboBox.Size = new System.Drawing.Size(305, 21);
    227227      this.colorComboBox.TabIndex = 13;
    228228      this.colorComboBox.SelectedIndexChanged += new System.EventHandler(this.colorComboBox_SelectedIndexChanged);
    … …  
    234234      this.sizeComboBox.Location = new System.Drawing.Point(287, 6);
    235235      this.sizeComboBox.Name = "sizeComboBox";
    236       this.sizeComboBox.Size = new System.Drawing.Size(222, 21);
     236      this.sizeComboBox.Size = new System.Drawing.Size(271, 21);
    237237      this.sizeComboBox.TabIndex = 13;
    238238      this.sizeComboBox.SelectedIndexChanged += new System.EventHandler(this.SizeComboBoxOnSelectedIndexChanged);
Note: See TracChangeset for help on using the changeset viewer.