source: branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem/3.3/Views/PerformanceModelingView.cs @ 13791

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using HeuristicLab.Common.Resources;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.MainForm;
using HeuristicLab.Optimization;
using HeuristicLab.OptimizationExpertSystem.Common;
using HeuristicLab.PluginInfrastructure;
using System;
using System.Collections.Generic;
using System.Linq;

namespace HeuristicLab.OptimizationExpertSystem {
  [View("Performance Modeling View")]
  [Content(typeof(KnowledgeCenter), IsDefaultView = false)]
  public partial class PerformanceModelingView : KnowledgeCenterViewBase {
    private readonly CheckedItemList<StringValue> characteristics;

    public PerformanceModelingView() {
      InitializeComponent();
      characteristics = new CheckedItemList<StringValue>();
      characteristics.CheckedItemsChanged += CharacteristicsOnCheckedItemsChanged;
      characteristicsViewHost.Content = characteristics;
      recommendStartButton.Text = string.Empty;
      recommendStartButton.Image = VSImageLibrary.Play;
      refreshCharacteristicsButton.Text = string.Empty;
      refreshCharacteristicsButton.Image = VSImageLibrary.Refresh;
      UpdateRecommenderCombobox();
    }

    protected override void OnContentChanged() {
      base.OnContentChanged();
      if (Content == null) {
        minTargetView.Content = null;
      } else {
        minTargetView.Content = Content.MinimumTarget;
      }
      UpdateCharacteristics();
    }

    protected override void SetEnabledStateOfControls() {
      base.SetEnabledStateOfControls();
      // TODO: up/download state
      recommendStartButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
      characteristicsViewHost.Enabled = Content != null;
      xValidateButton.Enabled = Content != null && recommenderComboBox.SelectedIndex >= 0 && characteristics.CheckedItems.Any();
    }

    #region Update Controls
    private void UpdateRecommenderCombobox() {
      if (InvokeRequired) { Invoke((Action)UpdateRecommenderCombobox); return; }
      var prevSelection = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
      var prevNewIndex = -1;
      recommenderComboBox.Items.Clear();

      var i = 0;
      foreach (var rcmd in ApplicationManager.Manager.GetInstances<IAlgorithmInstanceRecommender>()) {
        recommenderComboBox.Items.Add(rcmd);
        if (prevSelection == null || rcmd.GetType() == prevSelection.GetType())
          prevNewIndex = prevSelection == null ? 0 : i;
        i++;
      }
      
      recommenderComboBox.SelectedIndex = prevNewIndex;
    }

    private void UpdateCharacteristics() {
      if (InvokeRequired) { Invoke((Action)UpdateCharacteristics); return; }

      var @checked = new HashSet<string>(characteristics.CheckedItems.Select(x => x.Value.Value));
      if (@checked.Count == 0 && Content.ProblemInstances.ResultNames.Any(x => x.StartsWith("Characteristic.")))
        @checked = new HashSet<string>(Content.ProblemInstances.ResultNames.Where(x => x.StartsWith("Characteristic.")));

      characteristics.Clear();
      if (Content == null || Content.ProblemInstances.Count == 0) return;

      foreach (var c in Content.ProblemInstances.ResultNames) {
        characteristics.Add(new StringValue(c), @checked.Contains(c));
      }
    }
    #endregion

    #region Content Event Handlers
    protected override void OnProblemInstancesChanged() {
      base.OnProblemInstancesChanged();
      UpdateCharacteristics();
      SetEnabledStateOfControls();
    }
    #endregion

    #region Control Event Handlers
    private void RecommenderComboBoxOnSelectedIndexChanged(object sender, EventArgs e) {
      if (InvokeRequired) { Invoke((Action<object, EventArgs>)RecommenderComboBoxOnSelectedIndexChanged, sender, e); return; }
      var rcmd = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
      parameterCollectionView.Content = rcmd != null ? rcmd.Parameters : null;
      SetEnabledStateOfControls();
    }

    private void RecommendStartButtonOnClick(object sender, EventArgs e) {
      var rcmd = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();
      Content.RecommendationModel = rcmd.TrainModel(trainingSet, Content, characteristics.CheckedItems.Select(x => x.Value.Value).ToArray());
    }

    private void RefreshCharacteristicsButtonOnClick(object sender, EventArgs e) {
      UpdateCharacteristics();
      SetEnabledStateOfControls();
    }

    private void xValidateButton_Click(object sender, EventArgs e) {
      var recommender = (IAlgorithmInstanceRecommender)recommenderComboBox.SelectedItem;
      var features = characteristics.CheckedItems.Select(x => x.Value.Value).ToArray();

      var trainingSet = Content.ProblemInstances.Where(x => !Content.IsCurrentInstance(x)).ToArray();

      // leave one out crossvalidation
      foreach (var pi in trainingSet) {
        var model = recommender.TrainModel(trainingSet.Where(x => x != pi).ToArray(), Content, features);
        var ranking = model.GetRanking(pi).ToList();
        var realPerformance = Content.GetAlgorithmPerformance(pi);
        // absolute error predicting ert
        // confusion matrix predicting class
        // Kendall's tau
        // average NCDG ... relevance determined by clustering (unsuccessful algorithms being penalized with negative relevance)
        // mean reciprocal rank
        // optional: expected reciprocal rank
      }
    }

    private static double AbsoluteError(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
      var error = 0.0;
      foreach (var tuple in ranking) {
        double actual;
        if (!performance.TryGetValue(tuple.Item1, out actual)) continue;
        error += Math.Abs(actual - tuple.Item2);
      }
      return error;
    }

    private static int[,] ConfusionMatrix(Dictionary<IAlgorithm, double> performance, List<Tuple<IAlgorithm, double>> ranking) {
      var confMatrix = new int[5,5];
      return confMatrix;
    }
    #endregion

    #region Other Event Handlers
    private void CharacteristicsOnCheckedItemsChanged(object sender, EventArgs e) {
      SetEnabledStateOfControls();
    }
    #endregion
  }
}