source: branches/StatisticalTesting/HeuristicLab.Analysis.Statistics/3.3/StatisticalTestingView.cs @ 11271

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;
using HeuristicLab.Collections;
using HeuristicLab.Core.Views;
using HeuristicLab.Data;
using HeuristicLab.MainForm;
using HeuristicLab.Optimization;
using HeuristicLab.Optimization.Views;

namespace HeuristicLab.Analysis.Statistics {
  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")]
  [Content(typeof(RunCollection), false)]
  public sealed partial class StatisticalTestingView : ItemView {
    private const double significanceLevel = 0.05;
    private double[][] data;

    public StatisticalTestingView() {
      InitializeComponent();
    }

    public new RunCollection Content {
      get { return (RunCollection)base.Content; }
      set { base.Content = value; }
    }

    public override bool ReadOnly {
      get { return true; }
      set { /*not needed because results are always readonly */}
    }

    protected override void OnContentChanged() {
      base.OnContentChanged();

      if (Content != null) {
        UpdateResultComboBox();
        UpdateGroupsComboBox();
        FillCompComboBox();
        RebuildDataTable();
      }
      UpdateCaption();
    }

    private void UpdateCaption() {
      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
    }

    #region events
    protected override void RegisterContentEvents() {
      base.RegisterContentEvents();
      Content.ItemsAdded += new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
      Content.ItemsRemoved += new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
      Content.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
    }

    protected override void DeregisterContentEvents() {
      base.DeregisterContentEvents();
      Content.ItemsAdded -= new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
      Content.ItemsRemoved -= new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
      Content.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
    }

    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      RebuildDataTable();
    }

    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
      if (!Content.UpdateOfRunsInProgress) {
        RebuildDataTable();
      }
    }
    #endregion

    private void UpdateGroupsComboBox() {
      groupComboBox.Items.Clear();

      var parameters = (from run in Content
                        where run.Visible
                        from param in run.Parameters
                        select param.Key).Distinct().ToArray();

      foreach (var p in parameters) {
        var variations = (from run in Content
                          where run.Visible && run.Parameters.ContainsKey(p) &&
                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
                          select ((dynamic)run.Parameters[p]).Value).Distinct();

        if (variations.Count() > 1) {
          groupComboBox.Items.Add(p);
        }
      }

      if (groupComboBox.Items.Count > 0) {
        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
        //and takes a long time to group
        List<int> possibleIndizes = new List<int>();
        for (int i = 0; i < groupComboBox.Items.Count; i++) {
          if (groupComboBox.Items[i].ToString() != "Seed"
            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
            possibleIndizes.Add(i);
          }
        }

        if (possibleIndizes.Count > 0) {
          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
        } else {
          groupComboBox.SelectedItem = groupComboBox.Items[0];
        }
      }
    }

    private string[] GetColumnNames(IEnumerable<IRun> runs) {
      string parameterName = (string)groupComboBox.SelectedItem;
      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
    }

    private void UpdateResultComboBox() {
      resultComboBox.Items.Clear();
      var results = (from run in Content
                     where run.Visible
                     from result in run.Results
                     where result.Value is IntValue || result.Value is DoubleValue
                     select result.Key).Distinct().ToArray();

      resultComboBox.Items.AddRange(results);
      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
    }

    private void FillCompComboBox() {
      string parameterName = (string)groupComboBox.SelectedItem;
      if (parameterName != null) {
        string resultName = (string)resultComboBox.SelectedItem;
        if (resultName != null) {
          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
          var columnNames = GetColumnNames(runs).ToList();
          groupCompComboBox.Items.Clear();
          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
        }
      }
    }

    private void RebuildDataTable() {
      string parameterName = (string)groupComboBox.SelectedItem;
      if (parameterName != null) {
        string resultName = (string)resultComboBox.SelectedItem;

        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
        var columnNames = GetColumnNames(runs);
        var groups = GetGroups(columnNames, runs);
        data = new double[columnNames.Count()][];

        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
        dt.ColumnNames = columnNames;
        DataTable histogramDataTable = new DataTable(resultName);

        for (int i = 0; i < columnNames.Count(); i++) {
          int j = 0;
          data[i] = new double[groups[i].Count()];
          DataRow row = new DataRow(columnNames[i]);
          row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Histogram;
          histogramDataTable.Rows.Add(row);

          foreach (IRun run in groups[i]) {
            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
            data[i][j] = dt[j, i];
            row.Values.Add(dt[j, i]);
            j++;
          }
        }

        dataTableView.Content = histogramDataTable;
        stringConvertibleMatrixView.Content = dt;
      }
    }

    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
      string parameterName = (string)groupComboBox.SelectedItem;

      foreach (string cn in columnNames) {
        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
        runCols.Add(tmpRuns);
      }

      return runCols;
    }

    private void ResetUI() {
      normalityLabel.Image = null;
      groupCompLabel.Image = null;
      pairwiseLabel.Image = null;
      pValTextBox.Text = string.Empty;
      equalDistsTextBox.Text = string.Empty;
    }

    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
      RebuildDataTable();
      ResetUI();
      CalculateValues();
    }

    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
      FillCompComboBox();
      RebuildDataTable();
      ResetUI();
      CalculateValues();
    }

    private bool VerifyDataLength(bool showMessage) {
      if (data == null || data.Length == 0)
        return false;

      //alglib needs at least 5 samples for computation
      if (data.Any(x => x.Length <= 5)) {
        if (showMessage)
          MessageBox.Show(this, "You need to choose samples with a size greater 5.", "HeuristicLab", MessageBoxButtons.OK,
            MessageBoxIcon.Error);
        return false;
      }
      return true;
    }

    private void CalculateValues() {
      if (!VerifyDataLength(true))
        return;

      if (data != null) {
        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
          .AddOperationProgressToView(this, "Calculating...");

        string curItem = (string)groupCompComboBox.SelectedItem;
        Task.Factory.StartNew(() => CalculateValuesAsync(curItem));
      }
    }

    private void CalculateValuesAsync(string groupName) {
      TestAllGroups();
      CalculateNormality();
      CalculateNormalityDetails();
      CalculatePairwiseTest(groupName);
      CalculatePairwiseTestDetails(groupName);

      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
    }

    private void CalculatePairwise(string groupName) {
      if (!VerifyDataLength(false))
        return;

      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(this, "Calculating...");
      Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName));
    }

    private void CalculatePairwiseAsync(string groupName) {
      CalculatePairwiseTest(groupName);
      CalculatePairwiseTestDetails(groupName);

      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
    }

    private void TestAllGroups() {
      double pval = KruskalWallis.Test(data);
      pValTextBox.Text = pval.ToString();
      if (pval < significanceLevel) {
        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
      } else {
        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
      }
    }

    private void CalculateNormality() {
      double val;
      List<double> res = new List<double>();

      for (int i = 0; i < data.Length; i++) {
        alglib.jarqueberatest(data[i], data[i].Length, out val);
        res.Add(val);
      }

      // p-value is below significance level and thus the null hypothesis (data is normally distributed) is rejected. 
      if (res.Any(x => x < significanceLevel)) {
        this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
      } else {
        this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
      }
    }

    private void CalculateNormalityDetails() {
      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
      pValsMatrix.RowNames = new string[] { "p-Value" };

      double val;
      for (int i = 0; i < data.Length; i++) {
        alglib.jarqueberatest(data[i], data[i].Length, out val);
        pValsMatrix[0, i] = val;
      }

      this.Invoke(new Action(() => { normalityStringConvertibleMatrixView.Content = pValsMatrix; }));
    }

    private void CalculatePairwiseTest(string groupName) {
      int colIndex = 0;
      IEnumerable<string> columnNames = null;
      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));

      foreach (string col in columnNames) {
        if (col == groupName) {
          break;
        }
        colIndex++;
      }

      double[][] newData = FilterDataForPairwiseTest(colIndex);

      double mwuBothtails;
      double mwuLefttail;
      double mwuRighttail;
      int cnt = 0;

      for (int i = 0; i < newData.Length; i++) {
        alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, newData[i], newData[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
        if (mwuBothtails > significanceLevel) {
          cnt++;
        }
      }

      double ratio = ((double)cnt) / (data.Length - 1) * 100.0;
      equalDistsTextBox.Text = ratio.ToString() + " %";

      if (cnt == 0) {
        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
      } else {
        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
      }
    }

    private double[][] FilterDataForPairwiseTest(int columnToRemove) {
      double[][] newData = new double[data.Length - 1][];

      int i = 0;
      int l = 0;
      while (i < data.Length) {
        if (i != columnToRemove) {
          double[] row = new double[data[i].Length - 1];
          newData[l] = row;

          int j = 0, k = 0;
          while (j < row.Length) {
            if (i != columnToRemove) {
              newData[l][j] = data[i][k];
              j++;
              k++;
            } else {
              k++;
            }
          }
          i++;
          l++;
        } else {
          i++;
        }
      }
      return newData;
    }

    private void CalculatePairwiseTestDetails(string groupName) {
      int colIndex = 0;
      IEnumerable<string> columnNames = null;
      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));

      foreach (string col in columnNames) {
        if (col == groupName) {
          break;
        }
        colIndex++;
      }

      double[][] newData = FilterDataForPairwiseTest(colIndex);

      columnNames = columnNames.Where(x => x != groupName).ToList();

      var rowNames = new string[] { "p-Value of Mann-Whitney U", "Adjusted p-Value of Mann-Whitney U", 
            "p-Value of T-Test", "Adjusted p-Value of T-Test", "Necessary Sample Size for T-Test", "Cohen's d", "Hedges' g" };

      DoubleMatrix pValsMatrix = new DoubleMatrix(rowNames.Length, columnNames.Count());
      pValsMatrix.ColumnNames = columnNames;
      pValsMatrix.RowNames = rowNames;

      double mwuBothtails;
      double mwuLefttail;
      double mwuRighttail;
      double tTestLefttail;
      double[] mwuPValues = new double[newData.Length];
      double[] tTestPValues = new double[newData.Length];
      bool[] decision = null;
      double[] adjustedMwuPValues = null;
      double[] adjustedTtestPValues = null;

      for (int i = 0; i < newData.Length; i++) {
        if (i != colIndex) {
          alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, newData[i], newData[i].Length, out mwuBothtails,
            out mwuLefttail, out mwuRighttail);
          tTestLefttail = TTest.Test(data[colIndex], newData[i]);
          mwuPValues[i] = mwuBothtails;
          tTestPValues[i] = tTestLefttail;
        }
      }

      adjustedMwuPValues = BonferroniHolm.Calculate(significanceLevel, mwuPValues, out decision);
      adjustedTtestPValues = BonferroniHolm.Calculate(significanceLevel, tTestPValues, out decision);

      for (int i = 0; i < newData.Length; i++) {
        if (i != colIndex) {
          pValsMatrix[0, i] = mwuPValues[i];
          pValsMatrix[1, i] = adjustedMwuPValues[i];
          pValsMatrix[2, i] = tTestPValues[i];
          pValsMatrix[3, i] = adjustedTtestPValues[i];
          pValsMatrix[4, i] = TTest.GetOptimalSampleSize(data[colIndex], newData[i]);
          pValsMatrix[5, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], newData[i]);
          pValsMatrix[6, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], newData[i]);
        }
      }

      this.Invoke(new Action(() => { pairwiseStringConvertibleMatrixView.Content = pValsMatrix; }));
    }

    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
      boxplotView.Content = Content;
      // TODO: enable as soon as we move to HeuristicLab.Optimization.Views
      // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem;
      // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem;
      boxplotView.Show();
    }

    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
      string curItem = (string)groupCompComboBox.SelectedItem;
      CalculatePairwise(curItem);
    }
  }
}