Free cookie consent management tool by TermsFeed Policy Generator

source: branches/StatisticalTesting/HeuristicLab.Analysis.Statistics/3.3/StatisticalTestingView.cs @ 9923

Last change on this file since 9923 was 9923, checked in by ascheibe, 11 years ago

#2031 fixed cross threading errors
File size: 15.3 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Threading.Tasks;
26using System.Windows.Forms;
27using HeuristicLab.Core.Views;
28using HeuristicLab.Data;
29using HeuristicLab.MainForm;
30using HeuristicLab.Optimization;
31using HeuristicLab.Optimization.Views;
32
33namespace HeuristicLab.Analysis.Statistics {
34  [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")]
35  [Content(typeof(RunCollection), false)]
36  public sealed partial class StatisticalTestingView : ItemView {
37    private double[][] data;
38
39    public StatisticalTestingView() {
40      InitializeComponent();
41    }
42
43    public new RunCollection Content {
44      get { return (RunCollection)base.Content; }
45      set { base.Content = value; }
46    }
47
48    public override bool ReadOnly {
49      get { return true; }
50      set { /*not needed because results are always readonly */}
51    }
52
53    protected override void OnContentChanged() {
54      base.OnContentChanged();
55
56      if (Content != null) {
57        UpdateResultComboBox();
58        UpdateGroupsComboBox();
59        FillCompComboBox();
60        RebuildDataTable();
61      }
62      UpdateCaption();
63    }
64
65    private void UpdateCaption() {
66      Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType());
67    }
68
69    #region events
70    protected override void RegisterContentEvents() {
71      base.RegisterContentEvents();
72      Content.ItemsAdded += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
73      Content.ItemsRemoved += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
74      Content.CollectionReset += new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
75      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
76    }
77
78    protected override void DeregisterContentEvents() {
79      base.DeregisterContentEvents();
80      Content.ItemsAdded -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded);
81      Content.ItemsRemoved -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved);
82      Content.CollectionReset -= new HeuristicLab.Collections.CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset);
83      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
84    }
85
86    private void Content_CollectionReset(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
87      RebuildDataTable();
88    }
89
90    private void Content_ItemsRemoved(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
91      RebuildDataTable();
92    }
93
94    private void Content_ItemsAdded(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
95      RebuildDataTable();
96    }
97
98    void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
99      if (!Content.UpdateOfRunsInProgress) {
100        RebuildDataTable();
101      }
102    }
103    #endregion
104
105    private void UpdateGroupsComboBox() {
106      groupComboBox.Items.Clear();
107
108      var parameters = (from run in Content
109                        where run.Visible
110                        from param in run.Parameters
111                        select param.Key).Distinct().ToArray();
112
113      foreach (var p in parameters) {
114        var variations = (from run in Content
115                          where run.Visible && run.Parameters.ContainsKey(p) &&
116                          (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue ||
117                          run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue)
118                          select ((dynamic)run.Parameters[p]).Value).Distinct();
119
120        if (variations.Count() > 1) {
121          groupComboBox.Items.Add(p);
122        }
123      }
124
125      if (groupComboBox.Items.Count > 0) {
126        //try to select something different than "Seed" or "Algorithm Name" as this makes no sense
127        //and takes a long time to group
128        List<int> possibleIndizes = new List<int>();
129        for (int i = 0; i < groupComboBox.Items.Count; i++) {
130          if (groupComboBox.Items[i].ToString() != "Seed"
131            && groupComboBox.Items[i].ToString() != "Algorithm Name") {
132            possibleIndizes.Add(i);
133          }
134        }
135
136        if (possibleIndizes.Count > 0) {
137          groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()];
138        } else {
139          groupComboBox.SelectedItem = groupComboBox.Items[0];
140        }
141      }
142    }
143
144    private string[] GetColumnNames(IEnumerable<IRun> runs) {
145      string parameterName = (string)groupComboBox.SelectedItem;
146      var r = runs.Where(x => x.Parameters.ContainsKey(parameterName));
147      return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray();
148    }
149
150    private void UpdateResultComboBox() {
151      resultComboBox.Items.Clear();
152      var results = (from run in Content
153                     where run.Visible
154                     from result in run.Results
155                     where result.Value is IntValue || result.Value is DoubleValue
156                     select result.Key).Distinct().ToArray();
157
158      resultComboBox.Items.AddRange(results);
159      if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0];
160    }
161
162    private void FillCompComboBox() {
163      string parameterName = (string)groupComboBox.SelectedItem;
164      if (parameterName != null) {
165        string resultName = (string)resultComboBox.SelectedItem;
166        if (resultName != null) {
167          var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
168          var columnNames = GetColumnNames(runs).ToList();
169          groupCompComboBox.Items.Clear();
170          columnNames.ForEach(x => groupCompComboBox.Items.Add(x));
171          if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0];
172        }
173      }
174    }
175
176    private void RebuildDataTable() {
177      string parameterName = (string)groupComboBox.SelectedItem;
178      if (parameterName != null) {
179        string resultName = (string)resultComboBox.SelectedItem;
180
181        var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible);
182        var columnNames = GetColumnNames(runs);
183        var groups = GetGroups(columnNames, runs);
184        data = new double[columnNames.Count()][];
185
186        DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count());
187        dt.ColumnNames = columnNames;
188
189        int i = 0;
190        int j = 0;
191        foreach (string columnName in columnNames) {
192          j = 0;
193          data[i] = new double[groups[i].Count()];
194          foreach (IRun run in groups[i]) {
195            dt[j, i] = (double)((dynamic)run.Results[resultName]).Value;
196            data[i][j] = dt[j, i];
197            j++;
198          }
199          i++;
200        }
201
202        stringConvertibleMatrixView.Content = dt;
203      }
204    }
205
206    private List<IEnumerable<IRun>> GetGroups(string[] columnNames, IEnumerable<IRun> runs) {
207      List<IEnumerable<IRun>> runCols = new List<IEnumerable<IRun>>();
208      string parameterName = (string)groupComboBox.SelectedItem;
209
210      foreach (string cn in columnNames) {
211        var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn);
212        runCols.Add(tmpRuns);
213      }
214
215      return runCols;
216    }
217
218    private void ResetUI() {
219      normalityLabel.Image = null;
220      groupCompLabel.Image = null;
221      pairwiseLabel.Image = null;
222      pValTextBox.Text = string.Empty;
223      equalDistsTextBox.Text = string.Empty;
224    }
225
226    private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) {
227      RebuildDataTable();
228      ResetUI();
229      CalculateValues();
230    }
231
232    private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) {
233      FillCompComboBox();
234      RebuildDataTable();
235      ResetUI();
236      CalculateValues();
237    }
238
239    private void CalculateValues() {
240      if (data != null) {
241        MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>()
242          .AddOperationProgressToView(this, "Calculating...");
243
244        string curItem = (string)groupCompComboBox.SelectedItem;
245        Task.Factory.StartNew(() => CalculateValuesAsync(curItem));
246      }
247    }
248
249    private void CalculateValuesAsync(string groupName) {
250      TestAllGroups();
251      CalculateNormality();
252      CalculateNormalityDetails();
253      CalculatePairwiseTest(groupName);
254      CalculatePairwiseTestDetails(groupName);
255
256      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
257    }
258
259    private void CalculatePairwise(string groupName) {
260      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().AddOperationProgressToView(this, "Calculating...");
261      Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName));
262    }
263
264    private void CalculatePairwiseAsync(string groupName) {
265      CalculatePairwiseTest(groupName);
266      CalculatePairwiseTestDetails(groupName);
267
268      MainFormManager.GetMainForm<HeuristicLab.MainForm.WindowsForms.MainForm>().RemoveOperationProgressFromView(this);
269    }
270
271    private void TestAllGroups() {
272      double pval = KruskalWallis.Test(data);
273      pValTextBox.Text = pval.ToString();
274      if (pval < 0.05) {
275        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
276      } else {
277        this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
278      }
279    }
280
281    private void CalculateNormality() {
282      double val;
283      List<double> res = new List<double>();
284
285      for (int i = 0; i < data.Length; i++) {
286        alglib.jarqueberatest(data[i], data[i].Length, out val);
287        res.Add(val);
288      }
289
290      for (int i = 0; i < res.Count(); i++) {
291        if (res[i] < 0.1) {
292          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
293        } else {
294          this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
295        }
296      }
297    }
298
299    private void CalculateNormalityDetails() {
300      DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns);
301      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
302      pValsMatrix.RowNames = new string[] { "p-Value" };
303
304      double val;
305      for (int i = 0; i < data.Length; i++) {
306        alglib.jarqueberatest(data[i], data[i].Length, out val);
307        pValsMatrix[0, i] = val;
308      }
309
310      this.Invoke(new Action(() => { normalityStringConvertibleMatrixView.Content = pValsMatrix; }));
311    }
312
313    private void CalculatePairwiseTest(string groupName) {
314      int colIndex = 0;
315      IEnumerable<string> columnNames = null;
316      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));
317
318      foreach (string col in columnNames) {
319        if (col == groupName) {
320          break;
321        }
322        colIndex++;
323      }
324
325      double mwuBothtails;
326      double mwuLefttail;
327      double mwuRighttail;
328      int cnt = 0;
329
330      for (int i = 0; i < data.Length; i++) {
331        if (i != colIndex) {
332          alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
333          if (mwuBothtails > 0.05) {
334            cnt++;
335          }
336        }
337      }
338
339      double ratio = ((double)cnt) / (data.Length - 1) * 100.0;
340      equalDistsTextBox.Text = ratio.ToString() + " %";
341
342      if (cnt == 0) {
343        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; }));
344      } else {
345        this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; }));
346      }
347    }
348
349    private void CalculatePairwiseTestDetails(string groupName) {
350      int colIndex = 0;
351      IEnumerable<string> columnNames = null;
352      this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; }));
353
354      foreach (string col in columnNames) {
355        if (col == groupName) {
356          break;
357        }
358        colIndex++;
359      }
360
361      DoubleMatrix pValsMatrix = new DoubleMatrix(5, stringConvertibleMatrixView.Content.Columns);
362      pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames;
363      pValsMatrix.RowNames = new string[] { "p-Value of Mann-Whitney U", "p-Value of T-Test", "Necessary Sample Size for T-Test", "Cohen's d", "Hedges' g" };
364
365      double mwuBothtails;
366      double mwuLefttail;
367      double mwuRighttail;
368      double ttestLefttail;
369      for (int i = 0; i < data.Length; i++) {
370        alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, data[i], data[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail);
371        ttestLefttail = TTest.Test(data[colIndex], data[i]);
372        pValsMatrix[0, i] = mwuBothtails;
373        pValsMatrix[1, i] = ttestLefttail;
374        pValsMatrix[2, i] = TTest.GetOptimalSampleSize(data[colIndex], data[i]);
375        pValsMatrix[3, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], data[i]);
376        pValsMatrix[4, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], data[i]);
377      }
378
379      this.Invoke(new Action(() => { pairwiseStringConvertibleMatrixView.Content = pValsMatrix; }));
380    }
381
382    private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) {
383      RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView();
384      boxplotView.Content = Content;
385      // TODO: enable as soon as we move to HeuristicLab.Optimization.Views
386      // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem;
387      // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem;
388      boxplotView.Show();
389    }
390
391    private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) {
392      string curItem = (string)groupCompComboBox.SelectedItem;
393      CalculatePairwise(curItem);
394    }
395  }
396}
Note: See TracBrowser for help on using the repository browser.