source: branches/StatisticalTesting/HeuristicLab.Analysis.Statistics/3.3/StatisticalTestsView.cs @ 11693

Last change on this file since 11693 was 11693, checked in by ascheibe, 8 years ago

#2031 added dialog for configuring the SignificanceLevel and renamed view

File size: 18.3 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2014 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Threading.Tasks;
26using System.Windows.Forms;
27using HeuristicLab.Collections;
28using HeuristicLab.Common;
29using HeuristicLab.Core.Views;
30using HeuristicLab.Data;
31using HeuristicLab.MainForm;
32using HeuristicLab.Optimization;
33using HeuristicLab.Optimization.Views;
34
35namespace HeuristicLab.Analysis.Statistics {
36  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")]
37  [Content(typeof(RunCollection), false)]
38  public sealed partial class StatisticalTestsView : ItemView, IConfigureableView {
39    private double significanceLevel = 0.05;
40    private const int requiredSampleSize = 5;
41    private double[][] data;
42
43    public double SignificanceLevel {
44      get { return significanceLevel; }
45      set {
46        if (!significanceLevel.IsAlmost(value)) {
47          significanceLevel = value;
48          ResetUI();
49          CalculateValues();
50        }
51      }
52    }
53
54    public new RunCollection Content {
55      get { return (RunCollection)base.Content; }
56      set { base.Content = value; }
57    }
58
59    public override bool ReadOnly {
60      get { return true; }
61      set { /*not needed because results are always readonly */}
62    }
63
64    public StatisticalTestsView() {
65      InitializeComponent();
66    }
67
68    public void ShowConfiguration() {
69      using (StatisticalTestsConfigurationDialog dlg = new StatisticalTestsConfigurationDialog(this)) {
70        dlg.ShowDialog(this);
71      }
72    }
73
74    protected override void OnContentChanged() {
75      base.OnContentChanged();
76
77      if (Content != null) {
78        UpdateResultComboBox();
79        UpdateGroupsComboBox();
80        FillCompComboBox();
81        RebuildDataTable();
82      }
83      UpdateCaption();
84    }
85
86    private void UpdateCaption() {
87      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
88    }
89
90    #region events
91    protected override void RegisterContentEvents() {
92      base.RegisterContentEvents();
93      Content.ItemsAdded += new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
94      Content.ItemsRemoved += new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
95      Content.CollectionReset += new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
96      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
97    }
98
99    protected override void DeregisterContentEvents() {
100      base.DeregisterContentEvents();
101      Content.ItemsAdded -= new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
102      Content.ItemsRemoved -= new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
103      Content.CollectionReset -= new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
104      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
105    }
106
107    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
108      RebuildDataTable();
109    }
110
111    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
112      RebuildDataTable();
113    }
114
115    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
116      RebuildDataTable();
117    }
118
119    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
120      if (!Content.UpdateOfRunsInProgress) {
121        RebuildDataTable();
122      }
123    }
124    #endregion
125
126    private void UpdateGroupsComboBox() {
127      groupComboBox.Items.Clear();
128
129      var parameters = (from run in Content
130                        where run.Visible
131                        from param in run.Parameters
132                        select param.Key).Distinct().ToArray();
133
134      foreach (var p in parameters) {
135        var variations = (from run in Content
136                          where run.Visible && run.Parameters.ContainsKey(p) &&
137                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
138                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
139                          select ((dynamic)run.Parameters[p]).Value).Distinct();
140
141        if (variations.Count() > 1) {
142          groupComboBox.Items.Add(p);
143        }
144      }
145
146      if (groupComboBox.Items.Count > 0) {
147        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
148        //and takes a long time to group
149        List<int> possibleIndizes = new List<int>();
150        for (int i = 0; i < groupComboBox.Items.Count; i++) {
151          if (groupComboBox.Items[i].ToString() != "Seed"
152            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
153            possibleIndizes.Add(i);
154          }
155        }
156
157        if (possibleIndizes.Count > 0) {
158          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
159        } else {
160          groupComboBox.SelectedItem = groupComboBox.Items[0];
161        }
162      }
163    }
164
165    private string[] GetColumnNames(IEnumerable<IRun> runs) {
166      string parameterName = (string)groupComboBox.SelectedItem;
167      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
168      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
169    }
170
171    private void UpdateResultComboBox() {
172      resultComboBox.Items.Clear();
173      var results = (from run in Content
174                     where run.Visible
175                     from result in run.Results
176                     where result.Value is IntValue || result.Value is DoubleValue
177                     select result.Key).Distinct().ToArray();
178
179      resultComboBox.Items.AddRange(results);
180      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
181    }
182
183    private void FillCompComboBox() {
184      string parameterName = (string)groupComboBox.SelectedItem;
185      if (parameterName != null) {
186        string resultName = (string)resultComboBox.SelectedItem;
187        if (resultName != null) {
188          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
189          var columnNames = GetColumnNames(runs).ToList();
190          groupCompComboBox.Items.Clear();
191          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
192          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
193        }
194      }
195    }
196
197    private void RebuildDataTable() {
198      string parameterName = (string)groupComboBox.SelectedItem;
199      if (parameterName != null) {
200        string resultName = (string)resultComboBox.SelectedItem;
201
202        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
203        var columnNames = GetColumnNames(runs);
204        var groups = GetGroups(columnNames, runs);
205        data = new double[columnNames.Count()][];
206
207        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
208        dt.ColumnNames = columnNames;
209        DataTable histogramDataTable = new DataTable(resultName);
210
211        for (int i = 0; i < columnNames.Count(); i++) {
212          int j = 0;
213          data[i] = new double[groups[i].Count()];
214          DataRow row = new DataRow(columnNames[i]);
215          row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Histogram;
216          histogramDataTable.Rows.Add(row);
217
218          foreach (IRun run in groups[i]) {
219            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
220            data[i][j] = dt[j, i];
221            row.Values.Add(dt[j, i]);
222            j++;
223          }
224        }
225
226        GenerateChart(histogramDataTable);
227        stringConvertibleMatrixView.Content = dt;
228      }
229    }
230
231    private void GenerateChart(DataTable histogramTable) {
232      foreach (var row in histogramTable.Rows) {
233        histogramControl.AddPoints(row.Name, row.Values, true);
234      }
235    }
236
237    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
238      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
239      string parameterName = (string)groupComboBox.SelectedItem;
240
241      foreach (string cn in columnNames) {
242        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
243        runCols.Add(tmpRuns);
244      }
245
246      return runCols;
247    }
248
249    private void ResetUI() {
250      normalityLabel.Image = null;
251      groupCompLabel.Image = null;
252      pairwiseLabel.Image = null;
253      pValTextBox.Text = string.Empty;
254      equalDistsTextBox.Text = string.Empty;
255    }
256
257    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
258      RebuildDataTable();
259      ResetUI();
260      CalculateValues();
261    }
262
263    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
264      FillCompComboBox();
265      RebuildDataTable();
266      ResetUI();
267      CalculateValues();
268    }
269
270    private bool VerifyDataLength(bool showMessage) {
271      if (data == null || data.Length == 0)
272        return false;
273
274      //alglib needs at least 5 samples for computation
275      if (data.Any(x => x.Length <= requiredSampleSize)) {
276        if (showMessage)
277          MessageBox.Show(this, "You need to choose samples with a size greater 5.", "HeuristicLab", MessageBoxButtons.OK,
278            MessageBoxIcon.Error);
279        return false;
280      }
281      return true;
282    }
283
284    private void CalculateValues() {
285      if (!VerifyDataLength(true))
286        return;
287
288      if (data != null) {
289        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
290          .AddOperationProgressToView(this, "Calculating...");
291
292        string curItem = (string)groupCompComboBox.SelectedItem;
293        Task.Factory.StartNew(() => CalculateValuesAsync(curItem));
294      }
295    }
296
297    private void CalculateValuesAsync(string groupName) {
298      TestAllGroups();
299      CalculateNormality();
300      CalculateNormalityDetails();
301      CalculatePairwiseTest(groupName);
302      CalculatePairwiseTestDetails(groupName);
303
304      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
305    }
306
307    private void CalculatePairwise(string groupName) {
308      if (!VerifyDataLength(false))
309        return;
310
311      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(pairwiseTestGroupBox, "Calculating...");
312      Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName));
313    }
314
315    private void CalculatePairwiseAsync(string groupName) {
316      CalculatePairwiseTest(groupName);
317      CalculatePairwiseTestDetails(groupName);
318
319      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(pairwiseTestGroupBox);
320    }
321
322    private void TestAllGroups() {
323      double pval = KruskalWallisTest.Test(data);
324      pValTextBox.Text = pval.ToString();
325      if (pval < significanceLevel) {
326        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
327      } else {
328        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
329      }
330    }
331
332    private void CalculateNormality() {
333      double val;
334      List<double> res = new List<double>();
335
336      for (int i = 0; i < data.Length; i++) {
337        alglib.jarqueberatest(data[i], data[i].Length, out val);
338        res.Add(val);
339      }
340
341      // p-value is below significance level and thus the null hypothesis (data is normally distributed) is rejected.
342      if (res.Any(x => x < significanceLevel)) {
343        this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
344      } else {
345        this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
346      }
347    }
348
349    private void CalculateNormalityDetails() {
350      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
351      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
352      pValsMatrix.RowNames = new string[] { "p-Value" };
353
354      double val;
355      for (int i = 0; i < data.Length; i++) {
356        alglib.jarqueberatest(data[i], data[i].Length, out val);
357        pValsMatrix[0, i] = val;
358      }
359
360      this.Invoke(new Action(() => {
361        normalityStringConvertibleMatrixView.Content = pValsMatrix;
362        normalityStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
363      }));
364    }
365
366    private void CalculatePairwiseTest(string groupName) {
367      int colIndex = 0;
368      IEnumerable<string> columnNames = null;
369      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));
370
371      foreach (string col in columnNames) {
372        if (col == groupName) {
373          break;
374        }
375        colIndex++;
376      }
377
378      double[][] newData = FilterDataForPairwiseTest(colIndex);
379      int cnt = 0;
380      for (int i = 0; i < newData.Length; i++) {
381        double mwuBothtails = PairwiseTest.MannWhitneyUTest(data[colIndex], newData[i]);
382        if (mwuBothtails > significanceLevel) {
383          cnt++;
384        }
385      }
386
387      double ratio = ((double)cnt) / (data.Length - 1) * 100.0;
388      equalDistsTextBox.Text = ratio.ToString() + " %";
389
390      if (cnt == 0) {
391        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
392      } else {
393        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
394      }
395    }
396
397    private double[][] FilterDataForPairwiseTest(int columnToRemove) {
398      double[][] newData = new double[data.Length - 1][];
399
400      int i = 0;
401      int l = 0;
402      while (i < data.Length) {
403        if (i != columnToRemove) {
404          double[] row = new double[data[i].Length - 1];
405          newData[l] = row;
406
407          int j = 0, k = 0;
408          while (j < row.Length) {
409            if (i != columnToRemove) {
410              newData[l][j] = data[i][k];
411              j++;
412              k++;
413            } else {
414              k++;
415            }
416          }
417          i++;
418          l++;
419        } else {
420          i++;
421        }
422      }
423      return newData;
424    }
425
426    private void CalculatePairwiseTestDetails(string groupName) {
427      int colIndex = 0;
428      IEnumerable<string> columnNames = null;
429      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));
430
431      foreach (string col in columnNames) {
432        if (col == groupName) {
433          break;
434        }
435        colIndex++;
436      }
437
438      double[][] newData = FilterDataForPairwiseTest(colIndex);
439
440      columnNames = columnNames.Where(x => x != groupName).ToList();
441
442      var rowNames = new string[] { "p-Value of Mann-Whitney U", "Adjusted p-Value of Mann-Whitney U",
443            "p-Value of T-Test", "Adjusted p-Value of T-Test", "Cohen's d", "Hedges' g" };
444
445      DoubleMatrix pValsMatrix = new DoubleMatrix(rowNames.Length, columnNames.Count());
446      pValsMatrix.ColumnNames = columnNames;
447      pValsMatrix.RowNames = rowNames;
448
449      double mwuBothtails;
450      double tTestBothTails;
451      double[] mwuPValues = new double[newData.Length];
452      double[] tTestPValues = new double[newData.Length];
453      bool[] decision = null;
454      double[] adjustedMwuPValues = null;
455      double[] adjustedTtestPValues = null;
456
457      for (int i = 0; i < newData.Length; i++) {
458        mwuBothtails = PairwiseTest.MannWhitneyUTest(data[colIndex], newData[i]);
459        tTestBothTails = PairwiseTest.TTest(data[colIndex], newData[i]);
460        mwuPValues[i] = mwuBothtails;
461        tTestPValues[i] = tTestBothTails;
462      }
463
464      adjustedMwuPValues = BonferroniHolm.Calculate(significanceLevel, mwuPValues, out decision);
465      adjustedTtestPValues = BonferroniHolm.Calculate(significanceLevel, tTestPValues, out decision);
466
467      for (int i = 0; i < newData.Length; i++) {
468        pValsMatrix[0, i] = mwuPValues[i];
469        pValsMatrix[1, i] = adjustedMwuPValues[i];
470        pValsMatrix[2, i] = tTestPValues[i];
471        pValsMatrix[3, i] = adjustedTtestPValues[i];
472        pValsMatrix[4, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], newData[i]);
473        pValsMatrix[5, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], newData[i]);
474      }
475
476      this.Invoke(new Action(() => {
477        pairwiseStringConvertibleMatrixView.Content = pValsMatrix;
478        pairwiseStringConvertibleMatrixView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.AllCells);
479      }));
480    }
481
482    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
483      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
484      boxplotView.Content = Content;
485      // TODO: enable as soon as we move to HeuristicLab.Optimization.Views
486      // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem;
487      // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem;
488      boxplotView.Show();
489    }
490
491    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
492      string curItem = (string)groupCompComboBox.SelectedItem;
493      CalculatePairwise(curItem);
494    }
495  }
496}
Note: See TracBrowser for help on using the repository browser.