Changeset 15736


Ignore:
Timestamp:
02/07/18 20:55:57 (19 months ago)
Author:
abeham
Message:

#1614: updates from last night ...

Location:
branches/1614_GeneralizedQAP
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

    r15255 r15736  
    2020#endregion
    2121
     22using System;
     23using System.Collections.Generic;
     24using System.Drawing;
     25using System.IO;
     26using System.Linq;
     27using System.Threading;
     28using System.Threading.Tasks;
     29using HeuristicLab.Algorithms.DataAnalysis;
    2230using HeuristicLab.Analysis;
    2331using HeuristicLab.Analysis.SelfOrganizingMaps;
     
    3240using HeuristicLab.Problems.DataAnalysis;
    3341using HeuristicLab.Random;
    34 using System;
    35 using System.Collections.Generic;
    36 using System.Drawing;
    37 using System.IO;
    38 using System.Linq;
    39 using System.Threading;
    40 using System.Threading.Tasks;
    41 using HeuristicLab.Algorithms.DataAnalysis;
    4242using Algorithm = HeuristicLab.Clients.OKB.Administration.Algorithm;
    4343using Problem = HeuristicLab.Clients.OKB.Administration.Problem;
     
    296296
    297297    private static readonly HashSet<string> InterestingValueNames = new HashSet<string>() {
    298       "QualityPerEvaluations", "Problem Name", "Problem Type", "Algorithm Name", "Algorithm Type", "Maximization", "BestKnownQuality"
     298      "QualityPerEvaluations", "QualityPerClock", "Problem Name", "Problem Type", "Algorithm Name", "Algorithm Type", "Maximization", "BestKnownQuality"
    299299    };
    300300
     
    517517
    518518          IItem bkParam;
    519           if (!prob.Parameters.TryGetValue("BestKnownQuality", out bkParam) || !(bkParam is DoubleValue)) {
     519          if (!prob.Parameters.TryGetValue("BestKnownQuality", out bkParam) || !(bkParam is DoubleValue) || double.IsNaN(((DoubleValue)bkParam).Value)) {
    520520            var list = kvp.Value.SelectMany(x => x.Value)
    521               .Where(x => x.Results.ContainsKey("QualityPerEvaluations"))
    522               .Select(x => ((IndexedDataTable<double>)x.Results["QualityPerEvaluations"]).Rows.First().Values.Last().Item2);
     521              .Where(x => x.Results.ContainsKey("QualityPerClock"))
     522              .Select(x => (IndexedDataTable<double>)x.Results["QualityPerClock"])
     523              .Where(x => x.Rows.Count > 0 && x.Rows.First().Values.Count > 0)
     524              .Select(x => x.Rows.First().Values.Last().Item2);
    523525            if (!list.Any()) continue;
    524526            bkParam = new DoubleValue(maximization ? list.Max() : list.Min());
     
    765767      return knowledgeBase.Where(x => ((StringValue)x.Parameters["Problem Name"]).Value == ((StringValue)problemInstance.Parameters["Problem Name"]).Value)
    766768                          .GroupBy(x => algorithmId2AlgorithmInstanceMapping.GetByFirst(algorithmId2RunMapping.GetBySecond(x).Single()))
    767                           .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", target, Maximization).ExpectedRuntime);
     769                          .ToDictionary(x => x.Key, x => ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerClock", target, Maximization).ExpectedRuntime);
    768770    }
    769771
     
    773775      return knowledgeBase.Where(x => ((StringValue)x.Parameters["Problem Name"]).Value == ((StringValue)problemInstance.Parameters["Problem Name"]).Value)
    774776                          .GroupBy(x => algorithmId2AlgorithmInstanceMapping.GetByFirst(algorithmId2RunMapping.GetBySecond(x).Single()))
    775                           .ToDictionary(x => x.Key, x => Math.Log10(ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", target, Maximization).ExpectedRuntime));
     777                          .ToDictionary(x => x.Key, x => Math.Log10(ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerClock", target, Maximization).ExpectedRuntime));
    776778    }
    777779
     
    806808          var max = ((BoolValue)prob.Parameters["Maximization"]).Value;
    807809          var bkq = ((DoubleValue)prob.Parameters["BestKnownQuality"]).Value;
    808           var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime;
     810          var ert = ExpectedRuntimeHelper.CalculateErt(pr.ToList(), "QualityPerClock", GetTarget(bkq, target, max), max).ExpectedRuntime;
    809811          if (double.IsInfinity(ert)) ert = int.MaxValue;
    810812          ds.AddRow(new object[] { pr.Key }.Concat(f.Cast<object>()).Concat(new object[] { ert }));
     
    872874
    873875        var values = pr.GroupBy(x => algorithmId2RunMapping.GetBySecond(x).Single())
    874                        .ToDictionary(x => x.Key, x => Math.Log10(ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerEvaluations", GetTarget(bkq, target, max), max).ExpectedRuntime));
     876                       .ToDictionary(x => x.Key, x => Math.Log10(ExpectedRuntimeHelper.CalculateErt(x.ToList(), "QualityPerClock", GetTarget(bkq, target, max), max).ExpectedRuntime));
    875877        var ranks = ClusteringHelper<long>.Cluster(nClasses, values, x => double.IsInfinity(x.Value))
    876878          .GetByCluster().ToList();
  • branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem.Common/3.3/Recommenders/KNearestNeighborModel.cs

    r13878 r15736  
    2020#endregion
    2121
     22using System;
     23using System.Collections.Generic;
     24using System.Linq;
    2225using HeuristicLab.Collections;
    2326using HeuristicLab.Common;
    2427using HeuristicLab.Optimization;
    25 using System;
    26 using System.Collections.Generic;
    27 using System.Linq;
    2828
    2929namespace HeuristicLab.OptimizationExpertSystem.Common {
     
    5555        else feature[f] = (feature[f] - means[f]) / sdevs[f];
    5656      }
     57      var dist = features.Select((f, i) => Math.Sqrt(f.Select((g, j) => Math.Sqrt((g - feature[j]) * (g - feature[j]))).Sum())).ToList();
    5758      var nearestK = features.Select((f, i) => new { ProblemInstanceIndex = i, Feature = f })
    58                              .OrderBy(x => x.Feature.Select((f, i) => (f - feature[i]) * (f - feature[i])).Sum());
     59                             .OrderBy(x => dist[x.ProblemInstanceIndex]);
    5960     
    6061      var performances = new Dictionary<IAlgorithm, Performance>();
  • branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.Designer.cs

    r13803 r15736  
    5757      this.topNComboBox = new System.Windows.Forms.ComboBox();
    5858      this.topNlabel = new System.Windows.Forms.Label();
     59      this.ndcgLabel = new System.Windows.Forms.Label();
    5960      this.kendallsTauLabel = new System.Windows.Forms.Label();
    6061      this.predictedLabel = new System.Windows.Forms.Label();
    6162      this.absoluteLogErrorLabel = new System.Windows.Forms.Label();
    6263      this.absoluteErrorLabel = new System.Windows.Forms.Label();
     64      this.ncdgView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6365      this.kendallsTauView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6466      this.absoluteLogErrorView = new HeuristicLab.Data.Views.StringConvertibleValueView();
     
    6870      this.minTargetView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    6971      this.minimumTargetLabel = new System.Windows.Forms.Label();
    70       this.ncdgView = new HeuristicLab.Data.Views.StringConvertibleValueView();
    71       this.ndcgLabel = new System.Windows.Forms.Label();
     72      this.spearmansRhoView = new HeuristicLab.Data.Views.StringConvertibleValueView();
     73      this.spearmansRhoLabel = new System.Windows.Forms.Label();
    7274      this.tabControl.SuspendLayout();
    7375      this.characteristicsTabPage.SuspendLayout();
     
    186188      this.crossvalidationTabPage.Controls.Add(this.topNlabel);
    187189      this.crossvalidationTabPage.Controls.Add(this.ndcgLabel);
     190      this.crossvalidationTabPage.Controls.Add(this.spearmansRhoLabel);
    188191      this.crossvalidationTabPage.Controls.Add(this.kendallsTauLabel);
    189192      this.crossvalidationTabPage.Controls.Add(this.predictedLabel);
     
    191194      this.crossvalidationTabPage.Controls.Add(this.absoluteErrorLabel);
    192195      this.crossvalidationTabPage.Controls.Add(this.ncdgView);
     196      this.crossvalidationTabPage.Controls.Add(this.spearmansRhoView);
    193197      this.crossvalidationTabPage.Controls.Add(this.kendallsTauView);
    194198      this.crossvalidationTabPage.Controls.Add(this.absoluteLogErrorView);
     
    223227      this.topNlabel.Text = "Evaluate only best ranked:";
    224228      //
     229      // ndcgLabel
     230      //
     231      this.ndcgLabel.AutoSize = true;
     232      this.ndcgLabel.Location = new System.Drawing.Point(519, 165);
     233      this.ndcgLabel.Margin = new System.Windows.Forms.Padding(3);
     234      this.ndcgLabel.Name = "ndcgLabel";
     235      this.ndcgLabel.Size = new System.Drawing.Size(106, 13);
     236      this.ndcgLabel.TabIndex = 3;
     237      this.ndcgLabel.Text = "Mean NDCG (top N):";
     238      //
    225239      // kendallsTauLabel
    226240      //
     
    263277      this.absoluteErrorLabel.TabIndex = 3;
    264278      this.absoluteErrorLabel.Text = "Mean Absolute Error (top N):";
     279      //
     280      // ncdgView
     281      //
     282      this.ncdgView.Caption = "StringConvertibleValue View";
     283      this.ncdgView.Content = null;
     284      this.ncdgView.LabelVisible = false;
     285      this.ncdgView.Location = new System.Drawing.Point(503, 184);
     286      this.ncdgView.Name = "ncdgView";
     287      this.ncdgView.ReadOnly = true;
     288      this.ncdgView.Size = new System.Drawing.Size(194, 21);
     289      this.ncdgView.TabIndex = 2;
    265290      //
    266291      // kendallsTauView
     
    339364      this.minimumTargetLabel.Text = "Target:";
    340365      //
    341       // ncdgView
    342       //
    343       this.ncdgView.Caption = "StringConvertibleValue View";
    344       this.ncdgView.Content = null;
    345       this.ncdgView.LabelVisible = false;
    346       this.ncdgView.Location = new System.Drawing.Point(503, 184);
    347       this.ncdgView.Name = "ncdgView";
    348       this.ncdgView.ReadOnly = true;
    349       this.ncdgView.Size = new System.Drawing.Size(194, 21);
    350       this.ncdgView.TabIndex = 2;
    351       //
    352       // ndcgLabel
    353       //
    354       this.ndcgLabel.AutoSize = true;
    355       this.ndcgLabel.Location = new System.Drawing.Point(519, 165);
    356       this.ndcgLabel.Margin = new System.Windows.Forms.Padding(3);
    357       this.ndcgLabel.Name = "ndcgLabel";
    358       this.ndcgLabel.Size = new System.Drawing.Size(106, 13);
    359       this.ndcgLabel.TabIndex = 3;
    360       this.ndcgLabel.Text = "Mean NDCG (top N):";
     366      // spearmansRhoView
     367      //
     368      this.spearmansRhoView.Caption = "StringConvertibleValue View";
     369      this.spearmansRhoView.Content = null;
     370      this.spearmansRhoView.LabelVisible = false;
     371      this.spearmansRhoView.Location = new System.Drawing.Point(503, 303);
     372      this.spearmansRhoView.Name = "spearmansRhoView";
     373      this.spearmansRhoView.ReadOnly = true;
     374      this.spearmansRhoView.Size = new System.Drawing.Size(194, 21);
     375      this.spearmansRhoView.TabIndex = 2;
     376      //
     377      // spearmansRhoLabel
     378      //
     379      this.spearmansRhoLabel.AutoSize = true;
     380      this.spearmansRhoLabel.Location = new System.Drawing.Point(519, 284);
     381      this.spearmansRhoLabel.Margin = new System.Windows.Forms.Padding(3);
     382      this.spearmansRhoLabel.Name = "spearmansRhoLabel";
     383      this.spearmansRhoLabel.Size = new System.Drawing.Size(118, 13);
     384      this.spearmansRhoLabel.TabIndex = 3;
     385      this.spearmansRhoLabel.Text = "Mean Spearman\'s Rho:";
    361386      //
    362387      // PerformanceModelingView
     
    409434    private System.Windows.Forms.Label ndcgLabel;
    410435    private Data.Views.StringConvertibleValueView ncdgView;
     436    private System.Windows.Forms.Label spearmansRhoLabel;
     437    private Data.Views.StringConvertibleValueView spearmansRhoView;
    411438  }
    412439}
  • branches/1614_GeneralizedQAP/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs

    r13878 r15736  
    2020#endregion
    2121
     22using System;
     23using System.Collections.Generic;
     24using System.Linq;
     25using System.Threading.Tasks;
    2226using HeuristicLab.Analysis;
    2327using HeuristicLab.Common.Resources;
     
    2832using HeuristicLab.OptimizationExpertSystem.Common;
    2933using HeuristicLab.PluginInfrastructure;
    30 using System;
    31 using System.Collections.Generic;
    32 using System.Linq;
    33 using System.Threading.Tasks;
    3434
    3535namespace HeuristicLab.OptimizationExpertSystem {
     
    181181        var tau = 0.0;
    182182        var tauCnt = 0;
     183        var rho = 0.0;
     184        var rhoCnt = 0;
    183185        var ndcg = 0.0;
    184186        var ndcgCnt = 0;
     
    207209          }
    208210          var kt = KendallsTau(observed, predicted);
    209           if (!double.IsNaN(kt)) {
     211          if (!double.IsNaN(kt) && !double.IsInfinity(kt)) {
    210212            tau += kt;
    211213            tauCnt++;
     214          }
     215          var sr = SpearmansRho(observed, predicted);
     216          if (!double.IsNaN(sr)) {
     217            rho += sr;
     218            rhoCnt++;
    212219          }
    213220          var gain = NDCG(observed, model.GetRanking(pi).Take(topN).Select(x => x.Key).ToList());
     
    216223            ndcgCnt++;
    217224          }
    218           // mean reciprocal rank
    219           // optional: expected reciprocal rank
    220225          progress.ProgressValue = ++count / (double)trainingSet.Length;
    221226        }
     
    223228        absLogErr /= absLogErrCnt;
    224229        tau /= tauCnt;
     230        rho /= rhoCnt;
    225231        ndcg /= ndcgCnt;
    226232
     
    230236        confusionMatrixView.Content = new IntMatrix(confMatrix) {ColumnNames = description};
    231237        kendallsTauView.Content = new DoubleValue(tau);
     238        spearmansRhoView.Content = new DoubleValue(rho);
    232239        ncdgView.Content = new DoubleValue(ndcg);
    233240      } finally { progress.Finish(); }
     
    272279    }
    273280
     281    private static double SpearmansRho(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
     282      var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
     283      var n = commonKeys.Count;
     284      if (n == 0) return double.NaN;
     285
     286      var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
     287                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
     288                                  .OrderBy(x => x.Index).Select(x => (double)x.Rank).ToArray();
     289      var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
     290                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
     291                                     .OrderBy(x => x.Index).Select(x => (double)x.Rank).ToArray();
     292
     293      return alglib.spearmancorr2(actualRanks, predictedRanks);
     294    }
     295
    274296    private static double KendallsTau(Dictionary<IAlgorithm, double> performance, Dictionary<IAlgorithm, double> ranking) {
    275297      var commonKeys = performance.Keys.Intersect(ranking.Keys).ToList();
     
    278300
    279301      var actualRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = performance[x] })
    280                                   .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
     302                                  .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
    281303                                  .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
    282304      var predictedRanks = commonKeys.Select((x, i) => new { Alg = x, Index = i, Perf = ranking[x] })
    283                                      .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = i })
     305                                     .OrderBy(x => x.Perf).Select((x, i) => new { Index = x.Index, Rank = double.IsPositiveInfinity(x.Perf) ? int.MaxValue : i })
    284306                                     .OrderBy(x => x.Index).Select(x => x.Rank).ToList();
    285307
     
    292314            || paired[i].Actual < paired[j].Actual && paired[i].Predicted < paired[j].Predicted)
    293315            concordant++;
    294           else discordant++;
     316          else if (paired[i].Actual > paired[j].Actual && paired[i].Predicted < paired[j].Predicted
     317            || paired[i].Actual < paired[j].Actual && paired[i].Predicted > paired[j].Predicted)
     318            discordant++;
    295319        }
    296       return (2.0 * (concordant - discordant)) / (n * (n - 1));
     320
     321      var ti = performance.GroupBy(x => x.Value).Sum(x => x.Count() * (x.Count() - 1));
     322      var uj = ranking.GroupBy(x => x.Value).Sum(x => x.Count() * (x.Count() - 1));
     323      return (2.0 * (concordant - discordant)) / Math.Sqrt( (n * (n - 1) - ti * (ti - 1)) * (n * (n - 1) - uj * (uj - 1)) );
    297324    }
    298325
Note: See TracChangeset for help on using the changeset viewer.