source: branches/StatisticalTesting/HeuristicLab.Analysis.Statistics/3.3/StatisticalTestingView.cs @ 9922

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;
using HeuristicLab.Core.Views;
using HeuristicLab.Data;
using HeuristicLab.MainForm;
using HeuristicLab.Optimization;
using HeuristicLab.Optimization.Views;

namespace HeuristicLab.Analysis.Statistics {
  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")]
  [Content(typeof(RunCollection), false)]
  public sealed partial class StatisticalTestingView : ItemView {
    private double[][] data;

    public StatisticalTestingView() {
      InitializeComponent();
    }

    public new RunCollection Content {
      get { return (RunCollection)base.Content; }
      set { base.Content = value; }
    }

    public override bool ReadOnly {
      get { return true; }
      set { /*not needed because results are always readonly */}
    }

    protected override void OnContentChanged() {
      base.OnContentChanged();

      if (Content != null) {
        UpdateResultComboBox();
        UpdateGroupsComboBox();
        FillCompComboBox();
        RebuildDataTable();
      }
      UpdateCaption();
    }

    private void UpdateCaption() {
      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
    }

    #region events
    protected override void RegisterContentEvents() {
      base.RegisterContentEvents();
      Content.ItemsAdded += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
      Content.ItemsRemoved += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
      Content.CollectionReset += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
    }

    protected override void DeregisterContentEvents() {
      base.DeregisterContentEvents();
      Content.ItemsAdded -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
      Content.ItemsRemoved -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
      Content.CollectionReset -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
    }

    private void Content_CollectionReset(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    private void Content_ItemsRemoved(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    private void Content_ItemsAdded(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
      if (!Content.UpdateOfRunsInProgress) {
        RebuildDataTable();
      }
    }
    #endregion

    private void UpdateGroupsComboBox() {
      groupComboBox.Items.Clear();

      var parameters = (from run in Content
                        where run.Visible
                        from param in run.Parameters
                        select param.Key).Distinct().ToArray();

      foreach (var p in parameters) {
        var variations = (from run in Content
                          where run.Visible && run.Parameters.ContainsKey(p) &&
                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
                          select ((dynamic)run.Parameters[p]).Value).Distinct();

        if (variations.Count() > 1) {
          groupComboBox.Items.Add(p);
        }
      }

      if (groupComboBox.Items.Count > 0) {
        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
        //and takes a long time to group
        List<int> possibleIndizes = new List<int>();
        for (int i = 0; i < groupComboBox.Items.Count; i++) {
          if (groupComboBox.Items[i].ToString() != "Seed"
            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
            possibleIndizes.Add(i);
          }
        }

        if (possibleIndizes.Count > 0) {
          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
        } else {
          groupComboBox.SelectedItem = groupComboBox.Items[0];
        }
      }
    }

    private string[] GetColumnNames(IEnumerable<IRun> runs) {
      string parameterName = (string)groupComboBox.SelectedItem;
      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
    }

    private void UpdateResultComboBox() {
      resultComboBox.Items.Clear();
      var results = (from run in Content
                     where run.Visible
                     from result in run.Results
                     where result.Value is IntValue || result.Value is DoubleValue
                     select result.Key).Distinct().ToArray();

      resultComboBox.Items.AddRange(results);
      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
    }

    private void FillCompComboBox() {
      string parameterName = (string)groupComboBox.SelectedItem;
      if (parameterName != null) {
        string resultName = (string)resultComboBox.SelectedItem;
        if (resultName != null) {
          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
          var columnNames = GetColumnNames(runs).ToList();
          groupCompComboBox.Items.Clear();
          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
        }
      }
    }

    private void RebuildDataTable() {
      string parameterName = (string)groupComboBox.SelectedItem;
      if (parameterName != null) {
        string resultName = (string)resultComboBox.SelectedItem;

        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
        var columnNames = GetColumnNames(runs);
        var groups = GetGroups(columnNames, runs);
        data = new double[columnNames.Count()][];

        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
        dt.ColumnNames = columnNames;

        int i = 0;
        int j = 0;
        foreach (string columnName in columnNames) {
          j = 0;
          data[i] = new double[groups[i].Count()];
          foreach (IRun run in groups[i]) {
            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
            data[i][j] = dt[j, i];
            j++;
          }
          i++;
        }

        stringConvertibleMatrixView.Content = dt;
      }
    }

    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
      string parameterName = (string)groupComboBox.SelectedItem;

      foreach (string cn in columnNames) {
        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
        runCols.Add(tmpRuns);
      }

      return runCols;
    }

    private void ResetUI() {
      normalityLabel.Image = null;
      groupCompLabel.Image = null;
      pairwiseLabel.Image = null;
      pValTextBox.Text = string.Empty;
      equalDistsTextBox.Text = string.Empty;
    }

    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
      RebuildDataTable();
      ResetUI();
      CalculateValues();
    }

    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
      FillCompComboBox();
      RebuildDataTable();
      ResetUI();
      CalculateValues();
    }

    private void CalculateValues() {
      if (data != null) {
        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
          .AddOperationProgressToView(this, "Calculating...");
        Task.Factory.StartNew(CalculateValuesAsync);
      }
    }

    private void CalculateValuesAsync() {
      TestAllGroups();
      CalculateNormality();
      CalculateNormalityDetails();
      CalculatePairwiseTest();
      CalculatePairwiseTestDetails();

      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
    }

    private void CalculatePairwise() {
      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(this, "Calculating...");
      Task.Factory.StartNew(CalculatePairwiseAsync);
    }

    private void CalculatePairwiseAsync() {
      CalculatePairwiseTest();
      CalculatePairwiseTestDetails();

      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
    }

    private void TestAllGroups() {
      double pval = KruskalWallis.Test(data);
      pValTextBox.Text = pval.ToString();
      if (pval < 0.05) {
        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
      } else {
        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
      }
    }

    private void CalculateNormality() {
      double val;
      List<double> res = new List<double>();

      for (int i = 0; i < data.Length; i++) {
        alglib.jarqueberatest(data[i], data[i].Length, out val);
        res.Add(val);
      }

      for (int i = 0; i < res.Count(); i++) {
        if (res[i] < 0.1) {
          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
        } else {
          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
        }
      }
    }

    private void CalculateNormalityDetails() {
      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
      pValsMatrix.RowNames = new string[] { "p-Value" };

      double val;
      for (int i = 0; i < data.Length; i++) {
        alglib.jarqueberatest(data[i], data[i].Length, out val);
        pValsMatrix[0, i] = val;
      }

      this.Invoke(new Action(() => { normalityStringConvertibleMatrixView.Content = pValsMatrix; }));
    }

    private void CalculatePairwiseTest() {
      string curItem = (string)groupCompComboBox.SelectedItem;
      int colIndex = 0;

      foreach (string col in stringConvertibleMatrixView.Content.ColumnNames) {
        if (col == curItem) {
          break;
        }
        colIndex++;
      }

      double mwuBothtails;
      double mwuLefttail;
      double mwuRighttail;
      int cnt = 0;

      for (int i = 0; i < data.Length; i++) {
        if (i != colIndex) {
          alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
          if (mwuBothtails > 0.05) {
            cnt++;
          }
        }
      }

      double ratio = ((double)cnt) / (data.Length - 1) * 100.0;
      equalDistsTextBox.Text = ratio.ToString() + " %";

      if (cnt == 0) {
        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
      } else {
        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
      }
    }

    private void CalculatePairwiseTestDetails() {
      string curItem = (string)groupCompComboBox.SelectedItem;
      int colIndex = 0;

      foreach (string col in stringConvertibleMatrixView.Content.ColumnNames) {
        if (col == curItem) {
          break;
        }
        colIndex++;
      }

      DoubleMatrix pValsMatrix = new DoubleMatrix(5, stringConvertibleMatrixView.Content.Columns);
      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
      pValsMatrix.RowNames = new string[] { "p-Value of Mann-Whitney U", "p-Value of T-Test", "Necessary Sample Size for T-Test", "Cohen's d", "Hedges' g" };

      double mwuBothtails;
      double mwuLefttail;
      double mwuRighttail;
      double ttestLefttail;
      for (int i = 0; i < data.Length; i++) {
        alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
        ttestLefttail = TTest.Test(data[colIndex], data[i]);
        pValsMatrix[0, i] = mwuBothtails;
        pValsMatrix[1, i] = ttestLefttail;
        pValsMatrix[2, i] = TTest.GetOptimalSampleSize(data[colIndex], data[i]);
        pValsMatrix[3, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], data[i]);
        pValsMatrix[4, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], data[i]);
      }

      this.Invoke(new Action(() => { pairwiseStringConvertibleMatrixView.Content = pValsMatrix; }));
    }

    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
      boxplotView.Content = Content;
      // TODO: enable as soon as we move to HeuristicLab.Optimization.Views
      // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem;
      // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem;
      boxplotView.Show();
    }

    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
      CalculatePairwise();
    }
  }
}