Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Analysis.Statistics.Views/3.3/StatisticalTestsView.cs @ 12613

Last change on this file since 12613 was 12613, checked in by ascheibe, 9 years ago

#2270 fixed NullReferenceException in statistical tests view by introducing a flag to suppress firing ui events

File size: 18.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Threading.Tasks;
26using System.Windows.Forms;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core.Views;
30using HeuristicLab.Data;
31using HeuristicLab.MainForm;
32using HeuristicLab.Optimization;
33using HeuristicLab.Optimization.Views;
34
35namespace HeuristicLab.Analysis.Statistics.Views {
36  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.Views.InfoResources.StatisticalTestsInfo.rtf")]
37  [Content(typeof(RunCollection), false)]
38  public sealed partial class StatisticalTestsView : ItemView, IConfigureableView {
39    private double significanceLevel = 0.05;
40    private const int requiredSampleSize = 5;
41    private double[][] data;
42    private bool suppressUpdates = false;
43    private bool initializing = false;
44
45    public double SignificanceLevel {
46      get { return significanceLevel; }
47      set {
48        if (!significanceLevel.IsAlmost(value)) {
49          significanceLevel = value;
50          ResetUI();
51          CalculateValues();
52        }
53      }
54    }
55
56    public new RunCollection Content {
57      get { return (RunCollection)base.Content; }
58      set { base.Content = value; }
59    }
60
61    public override bool ReadOnly {
62      get { return true; }
63      set { /*not needed because results are always readonly */}
64    }
65
66    public StatisticalTestsView() {
67      InitializeComponent();
68    }
69
70    public void ShowConfiguration() {
71      using (StatisticalTestsConfigurationDialog dlg = new StatisticalTestsConfigurationDialog(this)) {
72        dlg.ShowDialog(this);
73      }
74    }
75
76    protected override void OnContentChanged() {
77      base.OnContentChanged();
78
79      if (Content != null) {
80        initializing = true;
81        UpdateResultComboBox();
82        UpdateGroupsComboBox();
83        RebuildDataTable();
84        FillCompComboBox();
85        ResetUI();
86        CalculateValues();
87        initializing = false;
88      }
89      UpdateCaption();
90    }
91
92    private void UpdateCaption() {
93      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
94    }
95
96    #region events
97    protected override void RegisterContentEvents() {
98      base.RegisterContentEvents();
99      Content.ColumnsChanged += Content_ColumnsChanged;
100      Content.RowsChanged += Content_RowsChanged;
101      Content.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
102      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
103    }
104
105    protected override void DeregisterContentEvents() {
106      base.DeregisterContentEvents();
107      Content.ColumnsChanged -= Content_ColumnsChanged;
108      Content.RowsChanged -= Content_RowsChanged;
109      Content.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
110      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
111    }
112
113    void Content_RowsChanged(object sender, EventArgs e) {
114      if (suppressUpdates) return;
115      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_RowsChanged, sender, e);
116      else {
117        RebuildDataTable();
118      }
119    }
120
121    void Content_ColumnsChanged(object sender, EventArgs e) {
122      if (suppressUpdates) return;
123      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_ColumnsChanged, sender, e);
124      else {
125        RebuildDataTable();
126      }
127    }
128
129    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
130      if (suppressUpdates) return;
131      if (InvokeRequired) Invoke((Action<object, CollectionItemsChangedEventArgs<IRun>>)Content_CollectionReset, sender, e);
132      else {
133        RebuildDataTable();
134      }
135    }
136
137    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
138      if (InvokeRequired) Invoke((Action<object, EventArgs>)Content_UpdateOfRunsInProgressChanged, sender, e);
139      else {
140        suppressUpdates = Content.UpdateOfRunsInProgress;
141        if (!suppressUpdates) RebuildDataTable();
142      }
143    }
144
145    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
146      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
147      boxplotView.Content = Content;
148      boxplotView.SetXAxis(groupComboBox.SelectedItem.ToString());
149      boxplotView.SetYAxis(resultComboBox.SelectedItem.ToString());
150
151      boxplotView.Show();
152    }
153
154    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
155      if (initializing) return;
156      string curItem = (string)groupCompComboBox.SelectedItem;
157      CalculatePairwise(curItem);
158    }
159
160    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
161      if (initializing) return;
162      RebuildDataTable();
163      ResetUI();
164      CalculateValues();
165    }
166
167    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
168      if (initializing) return;
169      RebuildDataTable();
170      FillCompComboBox();
171      ResetUI();
172      CalculateValues();
173    }
174    #endregion
175
176    private void UpdateGroupsComboBox() {
177      groupComboBox.Items.Clear();
178
179      var parameters = (from run in Content
180                        where run.Visible
181                        from param in run.Parameters
182                        select param.Key).Distinct().ToArray();
183
184      foreach (var p in parameters) {
185        var variations = (from run in Content
186                          where run.Visible && run.Parameters.ContainsKey(p) &&
187                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
188                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
189                          select ((dynamic)run.Parameters[p]).Value).Distinct();
190
191        if (variations.Count() > 1) {
192          groupComboBox.Items.Add(p);
193        }
194      }
195
196      if (groupComboBox.Items.Count > 0) {
197        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
198        //and takes a long time to group
199        List<int> possibleIndizes = new List<int>();
200        for (int i = 0; i < groupComboBox.Items.Count; i++) {
201          if (groupComboBox.Items[i].ToString() != "Seed"
202            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
203            possibleIndizes.Add(i);
204          }
205        }
206
207        if (possibleIndizes.Count > 0) {
208          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
209        } else {
210          groupComboBox.SelectedItem = groupComboBox.Items[0];
211        }
212      }
213    }
214
215    private string[] GetColumnNames(IEnumerable<IRun> runs) {
216      string parameterName = (string)groupComboBox.SelectedItem;
217      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
218      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
219    }
220
221    private void UpdateResultComboBox() {
222      resultComboBox.Items.Clear();
223      var results = (from run in Content
224                     where run.Visible
225                     from result in run.Results
226                     where result.Value is IntValue || result.Value is DoubleValue
227                     select result.Key).Distinct().ToArray();
228
229      resultComboBox.Items.AddRange(results);
230      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
231    }
232
233    private void FillCompComboBox() {
234      string parameterName = (string)groupComboBox.SelectedItem;
235      if (parameterName != null) {
236        string resultName = (string)resultComboBox.SelectedItem;
237        if (resultName != null) {
238          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
239          var columnNames = GetColumnNames(runs).ToList();
240          groupCompComboBox.Items.Clear();
241          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
242          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
243        }
244      }
245    }
246
247    private void RebuildDataTable() {
248      string parameterName = (string)groupComboBox.SelectedItem;
249      if (parameterName != null) {
250        string resultName = (string)resultComboBox.SelectedItem;
251
252        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
253        var columnNames = GetColumnNames(runs);
254        var groups = GetGroups(columnNames, runs);
255        data = new double[columnNames.Count()][];
256
257        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
258        dt.ColumnNames = columnNames;
259        DataTable histogramDataTable = new DataTable(resultName);
260
261        for (int i = 0; i < columnNames.Count(); i++) {
262          int j = 0;
263          data[i] = new double[groups[i].Count()];
264          DataRow row = new DataRow(columnNames[i]);
265          row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Histogram;
266          histogramDataTable.Rows.Add(row);
267
268          foreach (IRun run in groups[i]) {
269            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
270            data[i][j] = dt[j, i];
271            row.Values.Add(dt[j, i]);
272            j++;
273          }
274        }
275
276        GenerateChart(histogramDataTable);
277        stringConvertibleMatrixView.Content = dt;
278      }
279    }
280
281    private void GenerateChart(DataTable histogramTable) {
282      histogramControl.ClearPoints();
283      foreach (var row in histogramTable.Rows) {
284        histogramControl.AddPoints(row.Name, row.Values, true);
285      }
286    }
287
288    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
289      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
290      string parameterName = (string)groupComboBox.SelectedItem;
291
292      foreach (string cn in columnNames) {
293        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
294        runCols.Add(tmpRuns);
295      }
296
297      return runCols;
298    }
299
300    private void ResetUI() {
301      normalityLabel.Image = null;
302      normalityTextLabel.Text = string.Empty;
303      groupCompLabel.Image = null;
304      groupComTextLabel.Text = string.Empty;
305      pairwiseLabel.Image = null;
306      pairwiseTextLabel.Text = string.Empty;
307
308      pValTextBox.Text = string.Empty;
309      equalDistsTextBox.Text = string.Empty;
310    }
311
312    private bool VerifyDataLength(bool showMessage) {
313      if (data == null || data.Length == 0)
314        return false;
315
316      //alglib needs at least 5 samples for computation
317      if (data.Any(x => x.Length < requiredSampleSize)) {
318        if (showMessage)
319          MessageBox.Show(this, "You need at least " + requiredSampleSize
320            + " samples per group for computing hypothesis tests.", "HeuristicLab", MessageBoxButtons.OK,
321            MessageBoxIcon.Error);
322        return false;
323      }
324      return true;
325    }
326
327    private void CalculateValues() {
328      if (!VerifyDataLength(true))
329        return;
330
331      if (data != null && data.All(x => x != null)) {
332        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
333          .AddOperationProgressToView(this, "Calculating...");
334
335        string curItem = (string)groupCompComboBox.SelectedItem;
336        Task.Factory.StartNew(() => CalculateValuesAsync(curItem));
337      }
338    }
339
340    private void CalculateValuesAsync(string groupName) {
341      CalculateAllGroupsTest();
342      CalculateNormalityTest();
343      CalculatePairwiseTest(groupName);
344
345      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
346    }
347
348    private void CalculatePairwise(string groupName) {
349      if (!VerifyDataLength(false))
350        return;
351
352      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(pairwiseTestGroupBox, "Calculating...");
353      Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName));
354    }
355
356    private void CalculatePairwiseAsync(string groupName) {
357      CalculatePairwiseTest(groupName);
358
359      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(pairwiseTestGroupBox);
360    }
361
362    private void CalculateAllGroupsTest() {
363      double pval = KruskalWallisTest.Test(data);
364      pValTextBox.Text = pval.ToString();
365      if (pval < significanceLevel) {
366        this.Invoke(new Action(() => {
367          groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Default;
368          groupComTextLabel.Text = "There are groups with different distributions";
369        }));
370      } else {
371        this.Invoke(new Action(() => {
372          groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning;
373          groupComTextLabel.Text = "Groups have an equal distribution";
374        }));
375      }
376    }
377
378    private void CalculateNormalityTest() {
379      double val;
380      List<double> res = new List<double>();
381      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
382      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
383      pValsMatrix.RowNames = new string[] { "p-Value" };
384
385      for (int i = 0; i < data.Length; i++) {
386        alglib.jarqueberatest(data[i], data[i].Length, out val);
387        res.Add(val);
388        pValsMatrix[0, i] = val;
389      }
390
391      // p-value is below significance level and thus the null hypothesis (data is normally distributed) is rejected
392      if (res.Any(x => x < significanceLevel)) {
393        this.Invoke(new Action(() => {
394          normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning;
395          normalityTextLabel.Text = "Some groups may not be normally distributed";
396        }));
397      } else {
398        this.Invoke(new Action(() => {
399          normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Default;
400          normalityTextLabel.Text = "All sample data is normally distributed";
401        }));
402      }
403
404      this.Invoke(new Action(() => {
405        normalityStringConvertibleMatrixView.Content = pValsMatrix;
406        normalityStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
407      }));
408    }
409
410    private void ShowPairwiseResult(int nrOfEqualDistributions) {
411      double ratio = ((double)nrOfEqualDistributions) / (data.Length - 1) * 100.0;
412      equalDistsTextBox.Text = ratio.ToString() + " %";
413
414      if (nrOfEqualDistributions == 0) {
415        this.Invoke(new Action(() => {
416          pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Default;
417          pairwiseTextLabel.Text = "All groups have different distributions";
418        }));
419      } else {
420        this.Invoke(new Action(() => {
421          pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning;
422          pairwiseTextLabel.Text = "Some groups have equal distributions";
423        }));
424      }
425    }
426
427    private void CalculatePairwiseTest(string groupName) {
428      var columnNames = stringConvertibleMatrixView.Content.ColumnNames.ToList();
429      int colIndex = columnNames.IndexOf(groupName);
430      columnNames = columnNames.Where(x => x != groupName).ToList();
431
432      double[][] newData = FilterDataForPairwiseTest(colIndex);
433
434      var rowNames = new string[] { "p-Value of Mann-Whitney U", "Adjusted p-Value of Mann-Whitney U",
435            "p-Value of T-Test", "Adjusted p-Value of T-Test", "Cohen's d", "Hedges' g" };
436
437      DoubleMatrix pValsMatrix = new DoubleMatrix(rowNames.Length, columnNames.Count());
438      pValsMatrix.ColumnNames = columnNames;
439      pValsMatrix.RowNames = rowNames;
440
441      double mwuBothTails;
442      double tTestBothTails;
443      double[] mwuPValues = new double[newData.Length];
444      double[] tTestPValues = new double[newData.Length];
445      bool[] decision = null;
446      double[] adjustedMwuPValues = null;
447      double[] adjustedTtestPValues = null;
448      int cnt = 0;
449
450      for (int i = 0; i < newData.Length; i++) {
451        mwuBothTails = PairwiseTest.MannWhitneyUTest(data[colIndex], newData[i]);
452        tTestBothTails = PairwiseTest.TTest(data[colIndex], newData[i]);
453        mwuPValues[i] = mwuBothTails;
454        tTestPValues[i] = tTestBothTails;
455
456        if (mwuBothTails > significanceLevel) {
457          cnt++;
458        }
459      }
460
461      adjustedMwuPValues = BonferroniHolm.Calculate(significanceLevel, mwuPValues, out decision);
462      adjustedTtestPValues = BonferroniHolm.Calculate(significanceLevel, tTestPValues, out decision);
463
464      for (int i = 0; i < newData.Length; i++) {
465        pValsMatrix[0, i] = mwuPValues[i];
466        pValsMatrix[1, i] = adjustedMwuPValues[i];
467        pValsMatrix[2, i] = tTestPValues[i];
468        pValsMatrix[3, i] = adjustedTtestPValues[i];
469        pValsMatrix[4, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], newData[i]);
470        pValsMatrix[5, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], newData[i]);
471      }
472
473      this.Invoke(new Action(() => {
474        pairwiseStringConvertibleMatrixView.Content = pValsMatrix;
475        pairwiseStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
476      }));
477
478      ShowPairwiseResult(cnt);
479    }
480
481    private double[][] FilterDataForPairwiseTest(int columnToRemove) {
482      double[][] newData = new double[data.Length - 1][];
483
484      int i = 0;
485      int l = 0;
486      while (i < data.Length) {
487        if (i != columnToRemove) {
488          double[] row = new double[data[i].Length - 1];
489          newData[l] = row;
490
491          int j = 0, k = 0;
492          while (j < row.Length) {
493            if (i != columnToRemove) {
494              newData[l][j] = data[i][k];
495              j++;
496              k++;
497            } else {
498              k++;
499            }
500          }
501          i++;
502          l++;
503        } else {
504          i++;
505        }
506      }
507      return newData;
508    }
509  }
510}
Note: See TracBrowser for help on using the repository browser.