Free cookie consent management tool by TermsFeed Policy Generator

source: stable/HeuristicLab.Problems.DataAnalysis.Symbolic.Views/3.4/RunCollectionVariableImpactView.cs @ 17097

Last change on this file since 17097 was 17097, checked in by mkommend, 5 years ago

#2520: Merged 16565 - 16579 into stable.

File size: 16.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.ComponentModel;
25using System.Drawing;
26using System.Linq;
27using System.Windows.Forms;
28using HeuristicLab.Common;
29using HeuristicLab.Data;
30using HeuristicLab.MainForm;
31using HeuristicLab.MainForm.WindowsForms;
32using HeuristicLab.Optimization;
33
34namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Views {
35  [Content(typeof(RunCollection), false)]
36  [View("Variable Impacts")]
37  public sealed partial class RunCollectionVariableImpactView : AsynchronousContentView {
38    private const string variableImpactResultName = "Variable impacts";
39    private const string crossValidationFoldsResultName = "CrossValidation Folds";
40    private const string numberOfFoldsParameterName = "Folds";
41    public RunCollectionVariableImpactView() {
42      InitializeComponent();
43    }
44
45    public new RunCollection Content {
46      get { return (RunCollection)base.Content; }
47      set { base.Content = value; }
48    }
49
50    #region events
51    protected override void RegisterContentEvents() {
52      base.RegisterContentEvents();
53      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
54      Content.ItemsAdded += Content_ItemsAdded;
55      Content.ItemsRemoved += Content_ItemsRemoved;
56      Content.CollectionReset += Content_CollectionReset;
57      RegisterRunEvents(Content);
58    }
59    protected override void DeregisterContentEvents() {
60      base.RegisterContentEvents();
61      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
62      Content.ItemsAdded -= Content_ItemsAdded;
63      Content.ItemsRemoved -= Content_ItemsRemoved;
64      Content.CollectionReset -= Content_CollectionReset;
65      DeregisterRunEvents(Content);
66    }
67    private void RegisterRunEvents(IEnumerable<IRun> runs) {
68      foreach (IRun run in runs)
69        run.PropertyChanged += Run_PropertyChanged;
70    }
71    private void DeregisterRunEvents(IEnumerable<IRun> runs) {
72      foreach (IRun run in runs)
73        run.PropertyChanged -= Run_PropertyChanged;
74    }
75    private void Content_ItemsAdded(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
76      RegisterRunEvents(e.Items);
77      UpdateData();
78    }
79    private void Content_ItemsRemoved(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
80      DeregisterRunEvents(e.Items);
81      UpdateData();
82    }
83    private void Content_CollectionReset(object sender, HeuristicLab.Collections.CollectionItemsChangedEventArgs<IRun> e) {
84      DeregisterRunEvents(e.OldItems);
85      RegisterRunEvents(e.Items);
86      UpdateData();
87    }
88    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
89      if (!Content.UpdateOfRunsInProgress) UpdateData();
90    }
91    private void Run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
92      if (!Content.UpdateOfRunsInProgress && e.PropertyName == "Visible")
93        UpdateData();
94    }
95    #endregion
96
97    protected override void OnContentChanged() {
98      base.OnContentChanged();
99      this.UpdateData();
100    }
101
102    private void comboBox_SelectedValueChanged(object sender, EventArgs e) {
103      if (comboBox.SelectedItem != null) {
104        var visibleRuns = from run in Content where run.Visible select run;
105        if (comboBox.SelectedIndex == 0) {
106          var selectedFolds = from r in visibleRuns
107                              let foldCollection = (RunCollection)r.Results[crossValidationFoldsResultName]
108                              from run in foldCollection
109                              let name = (r.Name + " " + run.Name)
110                              select new { run, name };
111          matrixView.Content = CalculateVariableImpactMatrix(selectedFolds.Select(x => x.run).ToArray(), selectedFolds.Select(x => x.name).ToArray());
112        } else {
113          var selectedFolds = from r in visibleRuns
114                              let foldCollection = (RunCollection)r.Results[crossValidationFoldsResultName]
115                              let run = foldCollection.ElementAt(comboBox.SelectedIndex - 1)
116                              let name = (r.Name + " " + run.Name)
117                              select new { run, name };
118          matrixView.Content = CalculateVariableImpactMatrix(selectedFolds.Select(x => x.run).ToArray(), selectedFolds.Select(x => x.name).ToArray());
119        }
120      }
121    }
122
123
124    private void UpdateData() {
125      if (InvokeRequired) {
126        Invoke((Action)UpdateData);
127      } else {
128        if (Content != null) {
129          comboBox.Items.Clear();
130          comboBox.Enabled = false;
131          comboBox.Visible = false;
132          foldsLabel.Visible = false;
133          variableImpactsGroupBox.Dock = DockStyle.Fill;
134          var visibleRuns = Content.Where(r => r.Visible).ToArray();
135          if (visibleRuns.Length == 0) {
136            DisplayMessage("Run collection is empty.");
137          } else if (visibleRuns.All(r => r.Parameters.ContainsKey(numberOfFoldsParameterName))) {
138            // check if all runs are comparable (CV or normal runs)
139            CheckAndUpdateCvRuns();
140          } else if (visibleRuns.All(r => !r.Parameters.ContainsKey(numberOfFoldsParameterName))) {
141            CheckAndUpdateNormalRuns();
142          } else {
143            // there is a mix of CV and normal runs => show an error message
144            DisplayMessage("The run collection contains a mixture of normal runs and cross-validation runs. Variable impact calculation does not work in this case.");
145          }
146        }
147      }
148    }
149
150    private void CheckAndUpdateCvRuns() {
151      var visibleRuns = from run in Content where run.Visible select run;
152      var representativeRun = visibleRuns.First();
153      // make sure all runs have the same number of folds
154      int nFolds = ((IntValue)representativeRun.Parameters[numberOfFoldsParameterName]).Value;
155      if (visibleRuns.All(r => ((IntValue)r.Parameters[numberOfFoldsParameterName]).Value == nFolds)) {
156        var allFoldResults = visibleRuns.SelectMany(run => (RunCollection)run.Results[crossValidationFoldsResultName]);
157
158        // make sure each fold contains variable impacts
159        if (!allFoldResults.All(r => r.Results.ContainsKey(variableImpactResultName))) {
160          DisplayMessage("At least one of the runs does not contain a variable impact result.");
161        } else {
162          // make sure each of the runs has the same input variables
163          var allVariableNames = from run in allFoldResults
164                                 let varImpacts = (DoubleMatrix)run.Results[variableImpactResultName]
165                                 select varImpacts.RowNames;
166          var groupedVariableNames = allVariableNames
167            .SelectMany(x => x)
168            .GroupBy(x => x);
169
170          if (groupedVariableNames.Any(g => g.Count() != allFoldResults.Count())) {
171            DisplayMessage("At least one of the runs has a different input variable set than the rest.");
172          } else {
173            // populate combobox
174            comboBox.Items.Add("Overall");
175            for (int foldIndex = 0; foldIndex < nFolds; foldIndex++) {
176              comboBox.Items.Add("Fold " + foldIndex);
177            }
178            comboBox.SelectedIndex = 0;
179            comboBox.Enabled = true;
180            comboBox.Visible = true;
181            foldsLabel.Visible = true;
182            variableImpactsGroupBox.Controls.Clear();
183            variableImpactsGroupBox.Dock = DockStyle.None;
184            variableImpactsGroupBox.Anchor = AnchorStyles.Left | AnchorStyles.Top | AnchorStyles.Right |
185                                             AnchorStyles.Bottom;
186            variableImpactsGroupBox.Height = this.Height - comboBox.Height - 12;
187            variableImpactsGroupBox.Width = this.Width;
188            matrixView.Dock = DockStyle.Fill;
189            variableImpactsGroupBox.Controls.Add(matrixView);
190          }
191        }
192      } else {
193        DisplayMessage("At least on of the cross-validation runs has a different number of folds than the rest.");
194      }
195    }
196
197    private void CheckAndUpdateNormalRuns() {
198      // make sure all runs contain variable impact results
199      var visibleRuns = from run in Content where run.Visible select run;
200
201      if (!visibleRuns.All(r => r.Results.ContainsKey(variableImpactResultName))) {
202        DisplayMessage("At least one of the runs does not contain a variable impact result.");
203      } else {
204        // make sure each of the runs has the same input variables
205        var allVariableNames = from run in visibleRuns
206                               let varImpacts = (DoubleMatrix)run.Results[variableImpactResultName]
207                               select varImpacts.RowNames;
208        var groupedVariableNames = allVariableNames
209          .SelectMany(x => x)
210          .GroupBy(x => x);
211
212        if (groupedVariableNames.Any(g => g.Count() != visibleRuns.Count())) {
213          DisplayMessage("At least one of the runs has a different input variable set than the rest.");
214        } else {
215          if (!variableImpactsGroupBox.Controls.Contains(matrixView)) {
216            variableImpactsGroupBox.Controls.Clear();
217            matrixView.Dock = DockStyle.Fill;
218            variableImpactsGroupBox.Controls.Add(matrixView);
219          }
220          matrixView.Content = CalculateVariableImpactMatrix(visibleRuns.ToArray(), visibleRuns.Select(r => r.Name).ToArray());
221        }
222      }
223    }
224
225    private DoubleMatrix CalculateVariableImpactMatrix(IRun[] runs, string[] runNames) {
226      DoubleMatrix matrix = null;
227      IEnumerable<DoubleMatrix> allVariableImpacts = (from run in runs
228                                                      select run.Results[variableImpactResultName]).Cast<DoubleMatrix>();
229      IEnumerable<string> variableNames = (from variableImpact in allVariableImpacts
230                                           from variableName in variableImpact.RowNames
231                                           select variableName)
232                                          .Distinct();
233      // filter variableNames: only include names that have at least one non-zero value in a run
234      List<string> variableNamesList = (from variableName in variableNames
235                                        where GetVariableImpacts(variableName, allVariableImpacts).Any(x => !x.IsAlmost(0.0))
236                                        select variableName)
237                                       .ToList();
238
239      List<string> statictics = new List<string> { "Median Rank", "Mean", "StdDev", "pValue" };
240      List<string> columnNames = new List<string>(runNames);
241      columnNames.AddRange(statictics);
242      int numberOfRuns = runs.Length;
243
244      matrix = new DoubleMatrix(variableNamesList.Count, numberOfRuns + statictics.Count);
245      matrix.SortableView = true;
246      matrix.ColumnNames = columnNames;
247
248      // calculate statistics
249      List<List<double>> variableImpactsOverRuns = (from variableName in variableNamesList
250                                                    select GetVariableImpacts(variableName, allVariableImpacts).ToList())
251                                             .ToList();
252      List<List<double>> variableRanks = (from variableName in variableNamesList
253                                          select GetVariableImpactRanks(variableName, allVariableImpacts).ToList())
254                                      .ToList();
255      if (variableImpactsOverRuns.Count() > 0) {
256        // the variable with the worst median impact value is chosen as the reference variable
257        // this is problematic if all variables are relevant, however works often in practice
258        List<double> referenceImpacts = (from impacts in variableImpactsOverRuns
259                                         let avg = impacts.Median()
260                                         orderby avg
261                                         select impacts)
262                                         .First();
263        // for all variables
264        for (int row = 0; row < variableImpactsOverRuns.Count; row++) {
265          // median rank
266          matrix[row, numberOfRuns] = variableRanks[row].Median();
267          // also show mean and std.dev. of relative variable impacts to indicate the relative difference in impacts of variables
268          matrix[row, numberOfRuns + 1] = Math.Round(variableImpactsOverRuns[row].Average(), 3);
269          matrix[row, numberOfRuns + 2] = Math.Round(variableImpactsOverRuns[row].StandardDeviation(), 3);
270
271          double leftTail = 0; double rightTail = 0; double bothTails = 0;
272          // calc differences of impacts for current variable and reference variable
273          double[] z = new double[referenceImpacts.Count];
274          for (int i = 0; i < z.Length; i++) {
275            z[i] = variableImpactsOverRuns[row][i] - referenceImpacts[i];
276          }
277          // wilcoxon signed rank test is used because the impact values of two variables in a single run are not independent
278          alglib.wsr.wilcoxonsignedranktest(z, z.Length, 0, ref bothTails, ref leftTail, ref rightTail);
279          matrix[row, numberOfRuns + 3] = Math.Round(bothTails, 4);
280        }
281      }
282
283      // fill matrix with impacts from runs
284      for (int i = 0; i < runs.Length; i++) {
285        IRun run = runs[i];
286        DoubleMatrix runVariableImpacts = (DoubleMatrix)run.Results[variableImpactResultName];
287        for (int j = 0; j < runVariableImpacts.Rows; j++) {
288          int rowIndex = variableNamesList.FindIndex(s => s == runVariableImpacts.RowNames.ElementAt(j));
289          if (rowIndex > -1) {
290            matrix[rowIndex, i] = Math.Round(runVariableImpacts[j, 0], 3);
291          }
292        }
293      }
294      // sort by median
295      var sortedMatrix = (DoubleMatrix)matrix.Clone();
296      var sortedIndexes = from i in Enumerable.Range(0, sortedMatrix.Rows)
297                          orderby matrix[i, numberOfRuns]
298                          select i;
299
300      int targetIndex = 0;
301      foreach (var sourceIndex in sortedIndexes) {
302        for (int c = 0; c < matrix.Columns; c++)
303          sortedMatrix[targetIndex, c] = matrix[sourceIndex, c];
304        targetIndex++;
305      }
306      sortedMatrix.RowNames = sortedIndexes.Select(i => variableNamesList[i]);
307
308      return sortedMatrix;
309    }
310
311    private IEnumerable<double> GetVariableImpactRanks(string variableName, IEnumerable<DoubleMatrix> allVariableImpacts) {
312      foreach (DoubleMatrix runVariableImpacts in allVariableImpacts) {
313        // certainly not yet very efficient because ranks are computed multiple times for the same run
314        string[] variableNames = runVariableImpacts.RowNames.ToArray();
315        double[] values = (from row in Enumerable.Range(0, runVariableImpacts.Rows)
316                           select runVariableImpacts[row, 0] * -1)
317                          .ToArray();
318        Array.Sort(values, variableNames);
319        // calculate ranks
320        double[] ranks = new double[values.Length];
321        // check for tied ranks
322        int i = 0;
323        while (i < values.Length) {
324          ranks[i] = i + 1;
325          int j = i + 1;
326          while (j < values.Length && values[i].IsAlmost(values[j])) {
327            ranks[j] = ranks[i];
328            j++;
329          }
330          i = j;
331        }
332        int rankIndex = 0;
333        foreach (string rowVariableName in variableNames) {
334          if (rowVariableName == variableName)
335            yield return ranks[rankIndex];
336          rankIndex++;
337        }
338      }
339    }
340
341    private IEnumerable<double> GetVariableImpacts(string variableName, IEnumerable<DoubleMatrix> allVariableImpacts) {
342      foreach (DoubleMatrix runVariableImpacts in allVariableImpacts) {
343        int row = 0;
344        foreach (string rowName in runVariableImpacts.RowNames) {
345          if (rowName == variableName)
346            yield return runVariableImpacts[row, 0];
347          row++;
348        }
349      }
350    }
351
352    private void DisplayMessage(string message) {
353      variableImpactsGroupBox.Controls.Remove(matrixView);
354      var label = new Label { TextAlign = ContentAlignment.MiddleCenter, Text = message, Dock = DockStyle.Fill };
355      variableImpactsGroupBox.Controls.Add(label);
356    }
357  }
358}
Note: See TracBrowser for help on using the repository browser.