Free cookie consent management tool by TermsFeed Policy Generator

source: branches/StatisticalTesting/HeuristicLab.Analysis.Statistics/3.3/StatisticalTestingView.cs @ 9922

Last change on this file since 9922 was 9922, checked in by ascheibe, 9 years ago

#2031 adapted views to changes in help system

File size: 14.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Threading.Tasks;
26using System.Windows.Forms;
27using HeuristicLab.Core.Views;
28using HeuristicLab.Data;
29using HeuristicLab.MainForm;
30using HeuristicLab.Optimization;
31using HeuristicLab.Optimization.Views;
32
33namespace HeuristicLab.Analysis.Statistics {
34  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")]
35  [Content(typeof(RunCollection), false)]
36  public sealed partial class StatisticalTestingView : ItemView {
37    private double[][] data;
38
39    public StatisticalTestingView() {
40      InitializeComponent();
41    }
42
43    public new RunCollection Content {
44      get { return (RunCollection)base.Content; }
45      set { base.Content = value; }
46    }
47
48    public override bool ReadOnly {
49      get { return true; }
50      set { /*not needed because results are always readonly */}
51    }
52
53    protected override void OnContentChanged() {
54      base.OnContentChanged();
55
56      if (Content != null) {
57        UpdateResultComboBox();
58        UpdateGroupsComboBox();
59        FillCompComboBox();
60        RebuildDataTable();
61      }
62      UpdateCaption();
63    }
64
65    private void UpdateCaption() {
66      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
67    }
68
69    #region events
70    protected override void RegisterContentEvents() {
71      base.RegisterContentEvents();
72      Content.ItemsAdded += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
73      Content.ItemsRemoved += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
74      Content.CollectionReset += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
75      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
76    }
77
78    protected override void DeregisterContentEvents() {
79      base.DeregisterContentEvents();
80      Content.ItemsAdded -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
81      Content.ItemsRemoved -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
82      Content.CollectionReset -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
83      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
84    }
85
86    private void Content_CollectionReset(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
87      RebuildDataTable();
88    }
89
90    private void Content_ItemsRemoved(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
91      RebuildDataTable();
92    }
93
94    private void Content_ItemsAdded(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
95      RebuildDataTable();
96    }
97
98    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
99      if (!Content.UpdateOfRunsInProgress) {
100        RebuildDataTable();
101      }
102    }
103    #endregion
104
105    private void UpdateGroupsComboBox() {
106      groupComboBox.Items.Clear();
107
108      var parameters = (from run in Content
109                        where run.Visible
110                        from param in run.Parameters
111                        select param.Key).Distinct().ToArray();
112
113      foreach (var p in parameters) {
114        var variations = (from run in Content
115                          where run.Visible && run.Parameters.ContainsKey(p) &&
116                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
117                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
118                          select ((dynamic)run.Parameters[p]).Value).Distinct();
119
120        if (variations.Count() > 1) {
121          groupComboBox.Items.Add(p);
122        }
123      }
124
125      if (groupComboBox.Items.Count > 0) {
126        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
127        //and takes a long time to group
128        List<int> possibleIndizes = new List<int>();
129        for (int i = 0; i < groupComboBox.Items.Count; i++) {
130          if (groupComboBox.Items[i].ToString() != "Seed"
131            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
132            possibleIndizes.Add(i);
133          }
134        }
135
136        if (possibleIndizes.Count > 0) {
137          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
138        } else {
139          groupComboBox.SelectedItem = groupComboBox.Items[0];
140        }
141      }
142    }
143
144    private string[] GetColumnNames(IEnumerable<IRun> runs) {
145      string parameterName = (string)groupComboBox.SelectedItem;
146      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
147      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
148    }
149
150    private void UpdateResultComboBox() {
151      resultComboBox.Items.Clear();
152      var results = (from run in Content
153                     where run.Visible
154                     from result in run.Results
155                     where result.Value is IntValue || result.Value is DoubleValue
156                     select result.Key).Distinct().ToArray();
157
158      resultComboBox.Items.AddRange(results);
159      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
160    }
161
162    private void FillCompComboBox() {
163      string parameterName = (string)groupComboBox.SelectedItem;
164      if (parameterName != null) {
165        string resultName = (string)resultComboBox.SelectedItem;
166        if (resultName != null) {
167          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
168          var columnNames = GetColumnNames(runs).ToList();
169          groupCompComboBox.Items.Clear();
170          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
171          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
172        }
173      }
174    }
175
176    private void RebuildDataTable() {
177      string parameterName = (string)groupComboBox.SelectedItem;
178      if (parameterName != null) {
179        string resultName = (string)resultComboBox.SelectedItem;
180
181        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
182        var columnNames = GetColumnNames(runs);
183        var groups = GetGroups(columnNames, runs);
184        data = new double[columnNames.Count()][];
185
186        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
187        dt.ColumnNames = columnNames;
188
189        int i = 0;
190        int j = 0;
191        foreach (string columnName in columnNames) {
192          j = 0;
193          data[i] = new double[groups[i].Count()];
194          foreach (IRun run in groups[i]) {
195            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
196            data[i][j] = dt[j, i];
197            j++;
198          }
199          i++;
200        }
201
202        stringConvertibleMatrixView.Content = dt;
203      }
204    }
205
206    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
207      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
208      string parameterName = (string)groupComboBox.SelectedItem;
209
210      foreach (string cn in columnNames) {
211        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
212        runCols.Add(tmpRuns);
213      }
214
215      return runCols;
216    }
217
218    private void ResetUI() {
219      normalityLabel.Image = null;
220      groupCompLabel.Image = null;
221      pairwiseLabel.Image = null;
222      pValTextBox.Text = string.Empty;
223      equalDistsTextBox.Text = string.Empty;
224    }
225
226    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
227      RebuildDataTable();
228      ResetUI();
229      CalculateValues();
230    }
231
232    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
233      FillCompComboBox();
234      RebuildDataTable();
235      ResetUI();
236      CalculateValues();
237    }
238
239    private void CalculateValues() {
240      if (data != null) {
241        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
242          .AddOperationProgressToView(this, "Calculating...");
243        Task.Factory.StartNew(CalculateValuesAsync);
244      }
245    }
246
247    private void CalculateValuesAsync() {
248      TestAllGroups();
249      CalculateNormality();
250      CalculateNormalityDetails();
251      CalculatePairwiseTest();
252      CalculatePairwiseTestDetails();
253
254      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
255    }
256
257    private void CalculatePairwise() {
258      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(this, "Calculating...");
259      Task.Factory.StartNew(CalculatePairwiseAsync);
260    }
261
262    private void CalculatePairwiseAsync() {
263      CalculatePairwiseTest();
264      CalculatePairwiseTestDetails();
265
266      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
267    }
268
269    private void TestAllGroups() {
270      double pval = KruskalWallis.Test(data);
271      pValTextBox.Text = pval.ToString();
272      if (pval < 0.05) {
273        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
274      } else {
275        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
276      }
277    }
278
279    private void CalculateNormality() {
280      double val;
281      List<double> res = new List<double>();
282
283      for (int i = 0; i < data.Length; i++) {
284        alglib.jarqueberatest(data[i], data[i].Length, out val);
285        res.Add(val);
286      }
287
288      for (int i = 0; i < res.Count(); i++) {
289        if (res[i] < 0.1) {
290          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
291        } else {
292          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
293        }
294      }
295    }
296
297    private void CalculateNormalityDetails() {
298      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
299      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
300      pValsMatrix.RowNames = new string[] { "p-Value" };
301
302      double val;
303      for (int i = 0; i < data.Length; i++) {
304        alglib.jarqueberatest(data[i], data[i].Length, out val);
305        pValsMatrix[0, i] = val;
306      }
307
308      this.Invoke(new Action(() => { normalityStringConvertibleMatrixView.Content = pValsMatrix; }));
309    }
310
311    private void CalculatePairwiseTest() {
312      string curItem = (string)groupCompComboBox.SelectedItem;
313      int colIndex = 0;
314
315      foreach (string col in stringConvertibleMatrixView.Content.ColumnNames) {
316        if (col == curItem) {
317          break;
318        }
319        colIndex++;
320      }
321
322      double mwuBothtails;
323      double mwuLefttail;
324      double mwuRighttail;
325      int cnt = 0;
326
327      for (int i = 0; i < data.Length; i++) {
328        if (i != colIndex) {
329          alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
330          if (mwuBothtails > 0.05) {
331            cnt++;
332          }
333        }
334      }
335
336      double ratio = ((double)cnt) / (data.Length - 1) * 100.0;
337      equalDistsTextBox.Text = ratio.ToString() + " %";
338
339      if (cnt == 0) {
340        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
341      } else {
342        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
343      }
344    }
345
346    private void CalculatePairwiseTestDetails() {
347      string curItem = (string)groupCompComboBox.SelectedItem;
348      int colIndex = 0;
349
350      foreach (string col in stringConvertibleMatrixView.Content.ColumnNames) {
351        if (col == curItem) {
352          break;
353        }
354        colIndex++;
355      }
356
357      DoubleMatrix pValsMatrix = new DoubleMatrix(5, stringConvertibleMatrixView.Content.Columns);
358      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
359      pValsMatrix.RowNames = new string[] { "p-Value of Mann-Whitney U", "p-Value of T-Test", "Necessary Sample Size for T-Test", "Cohen's d", "Hedges' g" };
360
361      double mwuBothtails;
362      double mwuLefttail;
363      double mwuRighttail;
364      double ttestLefttail;
365      for (int i = 0; i < data.Length; i++) {
366        alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
367        ttestLefttail = TTest.Test(data[colIndex], data[i]);
368        pValsMatrix[0, i] = mwuBothtails;
369        pValsMatrix[1, i] = ttestLefttail;
370        pValsMatrix[2, i] = TTest.GetOptimalSampleSize(data[colIndex], data[i]);
371        pValsMatrix[3, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], data[i]);
372        pValsMatrix[4, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], data[i]);
373      }
374
375      this.Invoke(new Action(() => { pairwiseStringConvertibleMatrixView.Content = pValsMatrix; }));
376    }
377
378    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
379      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
380      boxplotView.Content = Content;
381      // TODO: enable as soon as we move to HeuristicLab.Optimization.Views
382      // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem;
383      // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem;
384      boxplotView.Show();
385    }
386
387    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
388      CalculatePairwise();
389    }
390  }
391}
Note: See TracBrowser for help on using the repository browser.