Free cookie consent management tool by TermsFeed Policy Generator

source: stable/HeuristicLab.Analysis.Statistics.Views/3.3/StatisticalTestsView.cs @ 18155

Last change on this file since 18155 was 17181, checked in by swagner, 5 years ago

#2875: Merged r17180 from trunk to stable

File size: 19.3 KB
RevLine 
[9353]1#region License Information
2/* HeuristicLab
[17181]3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[9353]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
[9913]25using System.Threading.Tasks;
[9353]26using System.Windows.Forms;
[9937]27using HeuristicLab.Collections;
[11693]28using HeuristicLab.Common;
[12725]29using HeuristicLab.Common.Resources;
[9353]30using HeuristicLab.Core.Views;
31using HeuristicLab.Data;
32using HeuristicLab.MainForm;
33using HeuristicLab.Optimization;
34using HeuristicLab.Optimization.Views;
35
[11705]36namespace HeuristicLab.Analysis.Statistics.Views {
37  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.Views.InfoResources.StatisticalTestsInfo.rtf")]
[9353]38  [Content(typeof(RunCollection), false)]
[11693]39  public sealed partial class StatisticalTestsView : ItemView, IConfigureableView {
40    private double significanceLevel = 0.05;
[11612]41    private const int requiredSampleSize = 5;
[9353]42    private double[][] data;
[12725]43    private bool suppressUpdates;
44    private bool initializing;
[9353]45
[11693]46    public double SignificanceLevel {
47      get { return significanceLevel; }
48      set {
49        if (!significanceLevel.IsAlmost(value)) {
50          significanceLevel = value;
51          ResetUI();
52          CalculateValues();
53        }
54      }
[9353]55    }
56
57    public new RunCollection Content {
58      get { return (RunCollection)base.Content; }
59      set { base.Content = value; }
60    }
61
62    public override bool ReadOnly {
63      get { return true; }
64      set { /*not needed because results are always readonly */}
65    }
66
[11693]67    public StatisticalTestsView() {
68      InitializeComponent();
69    }
70
71    public void ShowConfiguration() {
72      using (StatisticalTestsConfigurationDialog dlg = new StatisticalTestsConfigurationDialog(this)) {
73        dlg.ShowDialog(this);
74      }
75    }
76
[9353]77    protected override void OnContentChanged() {
78      base.OnContentChanged();
79
80      if (Content != null) {
[12725]81        UpdateUI();
82      } else {
83        ResetUI();
[9353]84      }
[9911]85      UpdateCaption();
[9353]86    }
87
[12725]88    private void UpdateUI() {
89      initializing = true;
90      UpdateResultComboBox();
91      UpdateGroupsComboBox();
92      RebuildDataTable();
93      FillCompComboBox();
94      ResetUI();
95      CalculateValues();
96      initializing = false;
97    }
98
[9911]99    private void UpdateCaption() {
[9913]100      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
[9911]101    }
102
[9353]103    #region events
104    protected override void RegisterContentEvents() {
105      base.RegisterContentEvents();
[11696]106      Content.ColumnsChanged += Content_ColumnsChanged;
107      Content.RowsChanged += Content_RowsChanged;
[12725]108      Content.CollectionReset += Content_CollectionReset;
[9911]109      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
[9353]110    }
111
112    protected override void DeregisterContentEvents() {
113      base.DeregisterContentEvents();
[11696]114      Content.ColumnsChanged -= Content_ColumnsChanged;
115      Content.RowsChanged -= Content_RowsChanged;
[12725]116      Content.CollectionReset -= Content_CollectionReset;
[9911]117      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
[9353]118    }
[9911]119
[11696]120    void Content_RowsChanged(object sender, EventArgs e) {
[12725]121      if (suppressUpdates) return;
122      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_RowsChanged, sender, e);
123      else {
124        UpdateUI();
125      }
[9911]126    }
127
[11696]128    void Content_ColumnsChanged(object sender, EventArgs e) {
[12725]129      if (suppressUpdates) return;
130      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_ColumnsChanged, sender, e);
131      else {
132        UpdateUI();
133      }
[9911]134    }
135
[11696]136    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
[12725]137      if (suppressUpdates) return;
138      if (InvokeRequired) Invoke((Action<object, CollectionItemsChangedEventArgs<IRun>>)Content_CollectionReset, sender, e);
139      else {
140        UpdateUI();
141      }
[9911]142    }
143
144    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
[12725]145      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_UpdateOfRunsInProgressChanged, sender, e);
146      else {
147        suppressUpdates = Content.UpdateOfRunsInProgress;
148        if (!suppressUpdates) UpdateUI();
[9911]149      }
150    }
[12725]151
152    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
153      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
154      boxplotView.Content = Content;
155      boxplotView.SetXAxis(groupComboBox.SelectedItem.ToString());
156      boxplotView.SetYAxis(resultComboBox.SelectedItem.ToString());
157
158      boxplotView.Show();
159    }
160
161    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
162      if (initializing || suppressUpdates) return;
163      string curItem = (string)groupCompComboBox.SelectedItem;
164      CalculatePairwise(curItem);
165    }
166
167    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
168      if (initializing || suppressUpdates) return;
169      RebuildDataTable();
170      ResetUI();
171      CalculateValues();
172    }
173
174    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
175      if (initializing || suppressUpdates) return;
176      RebuildDataTable();
177      FillCompComboBox();
178      ResetUI();
179      CalculateValues();
180    }
[9353]181    #endregion
182
183    private void UpdateGroupsComboBox() {
[12725]184      string selectedItem = (string)groupComboBox.SelectedItem;
185
[9353]186      groupComboBox.Items.Clear();
187      var parameters = (from run in Content
188                        where run.Visible
189                        from param in run.Parameters
190                        select param.Key).Distinct().ToArray();
191
192      foreach (var p in parameters) {
193        var variations = (from run in Content
194                          where run.Visible && run.Parameters.ContainsKey(p) &&
195                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
196                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
197                          select ((dynamic)run.Parameters[p]).Value).Distinct();
198
199        if (variations.Count() > 1) {
200          groupComboBox.Items.Add(p);
201        }
202      }
203
204      if (groupComboBox.Items.Count > 0) {
205        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
206        //and takes a long time to group
207        List<int> possibleIndizes = new List<int>();
208        for (int i = 0; i < groupComboBox.Items.Count; i++) {
209          if (groupComboBox.Items[i].ToString() != "Seed"
210            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
211            possibleIndizes.Add(i);
212          }
213        }
214
[12725]215        if (selectedItem != null && groupComboBox.Items.Contains(selectedItem)) {
216          groupComboBox.SelectedItem = selectedItem;
217        } else if (possibleIndizes.Count > 0) {
[9353]218          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
219        }
220      }
221    }
222
223    private string[] GetColumnNames(IEnumerable<IRun> runs) {
224      string parameterName = (string)groupComboBox.SelectedItem;
225      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
226      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
227    }
228
229    private void UpdateResultComboBox() {
[12725]230      string selectedItem = (string)resultComboBox.SelectedItem;
231
[9353]232      resultComboBox.Items.Clear();
233      var results = (from run in Content
234                     where run.Visible
235                     from result in run.Results
236                     where result.Value is IntValue || result.Value is DoubleValue
237                     select result.Key).Distinct().ToArray();
238
239      resultComboBox.Items.AddRange(results);
[12725]240
241      if (selectedItem != null && resultComboBox.Items.Contains(selectedItem)) {
242        resultComboBox.SelectedItem = selectedItem;
243      } else if (resultComboBox.Items.Count > 0) {
244        resultComboBox.SelectedItem = resultComboBox.Items[0];
245      }
[9353]246    }
247
[9389]248    private void FillCompComboBox() {
[12725]249      string selectedItem = (string)groupCompComboBox.SelectedItem;
[9389]250      string parameterName = (string)groupComboBox.SelectedItem;
251      if (parameterName != null) {
252        string resultName = (string)resultComboBox.SelectedItem;
253        if (resultName != null) {
254          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
255          var columnNames = GetColumnNames(runs).ToList();
256          groupCompComboBox.Items.Clear();
257          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
[12725]258          if (selectedItem != null && groupCompComboBox.Items.Contains(selectedItem)) {
259            groupCompComboBox.SelectedItem = selectedItem;
260          } else if (groupCompComboBox.Items.Count > 0) {
261            groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
262          }
[9389]263        }
264      }
265    }
266
[9353]267    private void RebuildDataTable() {
268      string parameterName = (string)groupComboBox.SelectedItem;
269      if (parameterName != null) {
270        string resultName = (string)resultComboBox.SelectedItem;
271
272        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
273        var columnNames = GetColumnNames(runs);
274        var groups = GetGroups(columnNames, runs);
275        data = new double[columnNames.Count()][];
276
[12928]277        if (!groups.Any() || !columnNames.Any()) {
278          return;
279        }
280
[9353]281        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
282        dt.ColumnNames = columnNames;
[9937]283        DataTable histogramDataTable = new DataTable(resultName);
[9353]284
[9937]285        for (int i = 0; i < columnNames.Count(); i++) {
286          int j = 0;
[9353]287          data[i] = new double[groups[i].Count()];
[9937]288          DataRow row = new DataRow(columnNames[i]);
289          row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Histogram;
290          histogramDataTable.Rows.Add(row);
291
[9353]292          foreach (IRun run in groups[i]) {
293            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
294            data[i][j] = dt[j, i];
[9937]295            row.Values.Add(dt[j, i]);
[9353]296            j++;
297          }
298        }
299
[11611]300        GenerateChart(histogramDataTable);
[9353]301        stringConvertibleMatrixView.Content = dt;
302      }
303    }
304
[11611]305    private void GenerateChart(DataTable histogramTable) {
[11919]306      histogramControl.ClearPoints();
[11611]307      foreach (var row in histogramTable.Rows) {
[11612]308        histogramControl.AddPoints(row.Name, row.Values, true);
[11611]309      }
310    }
311
[9353]312    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
313      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
314      string parameterName = (string)groupComboBox.SelectedItem;
315
316      foreach (string cn in columnNames) {
[12928]317        var tmpRuns = runs.Where(x =>
318        x.Parameters.ContainsKey(parameterName) &&
319        (((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn));
[9353]320        runCols.Add(tmpRuns);
321      }
322
323      return runCols;
324    }
325
[9389]326    private void ResetUI() {
327      normalityLabel.Image = null;
[11695]328      normalityTextLabel.Text = string.Empty;
[9389]329      groupCompLabel.Image = null;
[11695]330      groupComTextLabel.Text = string.Empty;
[9749]331      pairwiseLabel.Image = null;
[11695]332      pairwiseTextLabel.Text = string.Empty;
333
[9389]334      pValTextBox.Text = string.Empty;
335      equalDistsTextBox.Text = string.Empty;
336    }
337
[9937]338    private bool VerifyDataLength(bool showMessage) {
[12928]339      if (data == null || data.Length < 2)
[9937]340        return false;
341
342      //alglib needs at least 5 samples for computation
[12725]343      if (data.Any(x => x.Length < requiredSampleSize)) {
[9937]344        if (showMessage)
[11695]345          MessageBox.Show(this, "You need at least " + requiredSampleSize
346            + " samples per group for computing hypothesis tests.", "HeuristicLab", MessageBoxButtons.OK,
[9937]347            MessageBoxIcon.Error);
348        return false;
349      }
350      return true;
351    }
352
[9913]353    private void CalculateValues() {
[9937]354      if (!VerifyDataLength(true))
355        return;
356
[12725]357      if (data != null && data.All(x => x != null)) {
[17062]358        Progress.Show(this, "Calculating...", ProgressMode.Indeterminate);
[9923]359
360        string curItem = (string)groupCompComboBox.SelectedItem;
361        Task.Factory.StartNew(() => CalculateValuesAsync(curItem));
[9922]362      }
[9913]363    }
364
[9923]365    private void CalculateValuesAsync(string groupName) {
[11696]366      CalculateAllGroupsTest();
367      CalculateNormalityTest();
[9923]368      CalculatePairwiseTest(groupName);
[9913]369
[17062]370      Progress.Hide(this);
[9913]371    }
372
[9923]373    private void CalculatePairwise(string groupName) {
[12725]374      if (groupName == null) return;
[9937]375      if (!VerifyDataLength(false))
376        return;
377
[17062]378      Progress.ShowOnControl(pairwiseTestGroupBox, "Calculating...", ProgressMode.Indeterminate);
[9923]379      Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName));
[9913]380    }
381
[9923]382    private void CalculatePairwiseAsync(string groupName) {
383      CalculatePairwiseTest(groupName);
[9913]384
[17062]385      Progress.HideFromControl(pairwiseTestGroupBox);
[9913]386    }
387
[11696]388    private void CalculateAllGroupsTest() {
[11692]389      double pval = KruskalWallisTest.Test(data);
[12725]390      DisplayAllGroupsTextResults(pval);
391    }
392
393    private void DisplayAllGroupsTextResults(double pval) {
394      if (InvokeRequired) {
395        Invoke((Action<double>)DisplayAllGroupsTextResults, pval);
396      } else {
397        pValTextBox.Text = pval.ToString();
398        if (pval < significanceLevel) {
399          groupCompLabel.Image = VSImageLibrary.Default;
[11695]400          groupComTextLabel.Text = "There are groups with different distributions";
[12725]401        } else {
402          groupCompLabel.Image = VSImageLibrary.Warning;
[11695]403          groupComTextLabel.Text = "Groups have an equal distribution";
[12725]404        }
[9389]405      }
[9353]406    }
407
[11696]408    private void CalculateNormalityTest() {
[9353]409      double val;
410      List<double> res = new List<double>();
[11696]411      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
412      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
[12725]413      pValsMatrix.RowNames = new[] { "p-Value" };
[9353]414
415      for (int i = 0; i < data.Length; i++) {
416        alglib.jarqueberatest(data[i], data[i].Length, out val);
417        res.Add(val);
[11696]418        pValsMatrix[0, i] = val;
[9353]419      }
420
[11696]421      // p-value is below significance level and thus the null hypothesis (data is normally distributed) is rejected
[9950]422      if (res.Any(x => x < significanceLevel)) {
[12725]423        Invoke(new Action(() => {
424          normalityLabel.Image = VSImageLibrary.Warning;
[11695]425          normalityTextLabel.Text = "Some groups may not be normally distributed";
426        }));
[9936]427      } else {
[12725]428        Invoke(new Action(() => {
429          normalityLabel.Image = VSImageLibrary.Default;
[11695]430          normalityTextLabel.Text = "All sample data is normally distributed";
431        }));
[9353]432      }
433
[12725]434      Invoke(new Action(() => {
[11601]435        normalityStringConvertibleMatrixView.Content = pValsMatrix;
436        normalityStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
437      }));
[9353]438    }
439
[11696]440    private void ShowPairwiseResult(int nrOfEqualDistributions) {
441      double ratio = ((double)nrOfEqualDistributions) / (data.Length - 1) * 100.0;
[12725]442      equalDistsTextBox.Text = ratio + " %";
[9913]443
[11696]444      if (nrOfEqualDistributions == 0) {
[12725]445        Invoke(new Action(() => {
446          pairwiseLabel.Image = VSImageLibrary.Default;
[11695]447          pairwiseTextLabel.Text = "All groups have different distributions";
448        }));
[9913]449      } else {
[12725]450        Invoke(new Action(() => {
451          pairwiseLabel.Image = VSImageLibrary.Warning;
[11695]452          pairwiseTextLabel.Text = "Some groups have equal distributions";
453        }));
[9913]454      }
455    }
456
[11696]457    private void CalculatePairwiseTest(string groupName) {
458      var columnNames = stringConvertibleMatrixView.Content.ColumnNames.ToList();
459      int colIndex = columnNames.IndexOf(groupName);
460      columnNames = columnNames.Where(x => x != groupName).ToList();
[9957]461
462      double[][] newData = FilterDataForPairwiseTest(colIndex);
463
[12928]464      var rowNames = new[] { "p-Value of Mann-Whitney U", "Adjusted p-Value of Mann-Whitney U",
[11691]465            "p-Value of T-Test", "Adjusted p-Value of T-Test", "Cohen's d", "Hedges' g" };
[9950]466
[9957]467      DoubleMatrix pValsMatrix = new DoubleMatrix(rowNames.Length, columnNames.Count());
468      pValsMatrix.ColumnNames = columnNames;
[9950]469      pValsMatrix.RowNames = rowNames;
[9353]470
[11696]471      double mwuBothTails;
[11691]472      double tTestBothTails;
[9957]473      double[] mwuPValues = new double[newData.Length];
474      double[] tTestPValues = new double[newData.Length];
[9950]475      bool[] decision = null;
476      double[] adjustedMwuPValues = null;
477      double[] adjustedTtestPValues = null;
[11696]478      int cnt = 0;
[9950]479
[9957]480      for (int i = 0; i < newData.Length; i++) {
[11696]481        mwuBothTails = PairwiseTest.MannWhitneyUTest(data[colIndex], newData[i]);
[11692]482        tTestBothTails = PairwiseTest.TTest(data[colIndex], newData[i]);
[11696]483        mwuPValues[i] = mwuBothTails;
[11692]484        tTestPValues[i] = tTestBothTails;
[11696]485
486        if (mwuBothTails > significanceLevel) {
487          cnt++;
488        }
[9353]489      }
490
[9950]491      adjustedMwuPValues = BonferroniHolm.Calculate(significanceLevel, mwuPValues, out decision);
492      adjustedTtestPValues = BonferroniHolm.Calculate(significanceLevel, tTestPValues, out decision);
493
[9957]494      for (int i = 0; i < newData.Length; i++) {
[11692]495        pValsMatrix[0, i] = mwuPValues[i];
496        pValsMatrix[1, i] = adjustedMwuPValues[i];
497        pValsMatrix[2, i] = tTestPValues[i];
498        pValsMatrix[3, i] = adjustedTtestPValues[i];
499        pValsMatrix[4, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], newData[i]);
500        pValsMatrix[5, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], newData[i]);
[9950]501      }
502
[12725]503      Invoke(new Action(() => {
[11601]504        pairwiseStringConvertibleMatrixView.Content = pValsMatrix;
505        pairwiseStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
506      }));
[11696]507
508      ShowPairwiseResult(cnt);
[9353]509    }
510
[11696]511    private double[][] FilterDataForPairwiseTest(int columnToRemove) {
512      double[][] newData = new double[data.Length - 1][];
513
514      int i = 0;
515      int l = 0;
516      while (i < data.Length) {
517        if (i != columnToRemove) {
518          double[] row = new double[data[i].Length - 1];
519          newData[l] = row;
520
521          int j = 0, k = 0;
522          while (j < row.Length) {
523            if (i != columnToRemove) {
524              newData[l][j] = data[i][k];
525              j++;
526              k++;
527            } else {
528              k++;
529            }
530          }
531          i++;
532          l++;
533        } else {
534          i++;
535        }
536      }
537      return newData;
538    }
[9353]539  }
540}
Note: See TracBrowser for help on using the repository browser.