#region License Information /* HeuristicLab * Copyright (C) 2002-2014 Heuristic and Evolutionary Algorithms Laboratory (HEAL) * * This file is part of HeuristicLab. * * HeuristicLab is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * HeuristicLab is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with HeuristicLab. If not, see . */ #endregion using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Windows.Forms; using HeuristicLab.Collections; using HeuristicLab.Core.Views; using HeuristicLab.Data; using HeuristicLab.MainForm; using HeuristicLab.Optimization; using HeuristicLab.Optimization.Views; namespace HeuristicLab.Analysis.Statistics { [View("Statistical Tests", "HeuristicLab.Analysis.Statistics.InfoResources.StatisticalTestsInfo.rtf")] [Content(typeof(RunCollection), false)] public sealed partial class StatisticalTestingView : ItemView { private const double significanceLevel = 0.05; private double[][] data; public StatisticalTestingView() { InitializeComponent(); } public new RunCollection Content { get { return (RunCollection)base.Content; } set { base.Content = value; } } public override bool ReadOnly { get { return true; } set { /*not needed because results are always readonly */} } protected override void OnContentChanged() { base.OnContentChanged(); if (Content != null) { UpdateResultComboBox(); UpdateGroupsComboBox(); FillCompComboBox(); RebuildDataTable(); } UpdateCaption(); } private void UpdateCaption() { Caption = Content != null ? Content.OptimizerName + " Statistical Tests" : ViewAttribute.GetViewName(GetType()); } #region events protected override void RegisterContentEvents() { base.RegisterContentEvents(); Content.ItemsAdded += new CollectionItemsChangedEventHandler(Content_ItemsAdded); Content.ItemsRemoved += new CollectionItemsChangedEventHandler(Content_ItemsRemoved); Content.CollectionReset += new CollectionItemsChangedEventHandler(Content_CollectionReset); Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged; } protected override void DeregisterContentEvents() { base.DeregisterContentEvents(); Content.ItemsAdded -= new CollectionItemsChangedEventHandler(Content_ItemsAdded); Content.ItemsRemoved -= new CollectionItemsChangedEventHandler(Content_ItemsRemoved); Content.CollectionReset -= new CollectionItemsChangedEventHandler(Content_CollectionReset); Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged; } private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs e) { RebuildDataTable(); } private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs e) { RebuildDataTable(); } private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs e) { RebuildDataTable(); } void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) { if (!Content.UpdateOfRunsInProgress) { RebuildDataTable(); } } #endregion private void UpdateGroupsComboBox() { groupComboBox.Items.Clear(); var parameters = (from run in Content where run.Visible from param in run.Parameters select param.Key).Distinct().ToArray(); foreach (var p in parameters) { var variations = (from run in Content where run.Visible && run.Parameters.ContainsKey(p) && (run.Parameters[p] is IntValue || run.Parameters[p] is DoubleValue || run.Parameters[p] is StringValue || run.Parameters[p] is BoolValue) select ((dynamic)run.Parameters[p]).Value).Distinct(); if (variations.Count() > 1) { groupComboBox.Items.Add(p); } } if (groupComboBox.Items.Count > 0) { //try to select something different than "Seed" or "Algorithm Name" as this makes no sense //and takes a long time to group List possibleIndizes = new List(); for (int i = 0; i < groupComboBox.Items.Count; i++) { if (groupComboBox.Items[i].ToString() != "Seed" && groupComboBox.Items[i].ToString() != "Algorithm Name") { possibleIndizes.Add(i); } } if (possibleIndizes.Count > 0) { groupComboBox.SelectedItem = groupComboBox.Items[possibleIndizes.First()]; } else { groupComboBox.SelectedItem = groupComboBox.Items[0]; } } } private string[] GetColumnNames(IEnumerable runs) { string parameterName = (string)groupComboBox.SelectedItem; var r = runs.Where(x => x.Parameters.ContainsKey(parameterName)); return r.Select(x => ((dynamic)x.Parameters[parameterName]).Value).Distinct().Select(x => (string)x.ToString()).ToArray(); } private void UpdateResultComboBox() { resultComboBox.Items.Clear(); var results = (from run in Content where run.Visible from result in run.Results where result.Value is IntValue || result.Value is DoubleValue select result.Key).Distinct().ToArray(); resultComboBox.Items.AddRange(results); if (resultComboBox.Items.Count > 0) resultComboBox.SelectedItem = resultComboBox.Items[0]; } private void FillCompComboBox() { string parameterName = (string)groupComboBox.SelectedItem; if (parameterName != null) { string resultName = (string)resultComboBox.SelectedItem; if (resultName != null) { var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible); var columnNames = GetColumnNames(runs).ToList(); groupCompComboBox.Items.Clear(); columnNames.ForEach(x => groupCompComboBox.Items.Add(x)); if (groupCompComboBox.Items.Count > 0) groupCompComboBox.SelectedItem = groupCompComboBox.Items[0]; } } } private void RebuildDataTable() { string parameterName = (string)groupComboBox.SelectedItem; if (parameterName != null) { string resultName = (string)resultComboBox.SelectedItem; var runs = Content.Where(x => x.Results.ContainsKey(resultName) && x.Visible); var columnNames = GetColumnNames(runs); var groups = GetGroups(columnNames, runs); data = new double[columnNames.Count()][]; DoubleMatrix dt = new DoubleMatrix(groups.Select(x => x.Count()).Max(), columnNames.Count()); dt.ColumnNames = columnNames; DataTable histogramDataTable = new DataTable(resultName); for (int i = 0; i < columnNames.Count(); i++) { int j = 0; data[i] = new double[groups[i].Count()]; DataRow row = new DataRow(columnNames[i]); row.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Histogram; histogramDataTable.Rows.Add(row); foreach (IRun run in groups[i]) { dt[j, i] = (double)((dynamic)run.Results[resultName]).Value; data[i][j] = dt[j, i]; row.Values.Add(dt[j, i]); j++; } } dataTableView.Content = histogramDataTable; stringConvertibleMatrixView.Content = dt; } } private List> GetGroups(string[] columnNames, IEnumerable runs) { List> runCols = new List>(); string parameterName = (string)groupComboBox.SelectedItem; foreach (string cn in columnNames) { var tmpRuns = runs.Where(x => ((string)((dynamic)x.Parameters[parameterName]).Value.ToString()) == cn); runCols.Add(tmpRuns); } return runCols; } private void ResetUI() { normalityLabel.Image = null; groupCompLabel.Image = null; pairwiseLabel.Image = null; pValTextBox.Text = string.Empty; equalDistsTextBox.Text = string.Empty; } private void resultComboBox_SelectedValueChanged(object sender, EventArgs e) { RebuildDataTable(); ResetUI(); CalculateValues(); } private void groupComboBox_SelectedValueChanged(object sender, EventArgs e) { FillCompComboBox(); RebuildDataTable(); ResetUI(); CalculateValues(); } private bool VerifyDataLength(bool showMessage) { if (data == null || data.Length == 0) return false; //alglib needs at least 5 samples for computation if (data.Any(x => x.Length <= 5)) { if (showMessage) MessageBox.Show(this, "You need to choose samples with a size greater 5.", "HeuristicLab", MessageBoxButtons.OK, MessageBoxIcon.Error); return false; } return true; } private void CalculateValues() { if (!VerifyDataLength(true)) return; if (data != null) { MainFormManager.GetMainForm() .AddOperationProgressToView(this, "Calculating..."); string curItem = (string)groupCompComboBox.SelectedItem; Task.Factory.StartNew(() => CalculateValuesAsync(curItem)); } } private void CalculateValuesAsync(string groupName) { TestAllGroups(); CalculateNormality(); CalculateNormalityDetails(); CalculatePairwiseTest(groupName); CalculatePairwiseTestDetails(groupName); MainFormManager.GetMainForm().RemoveOperationProgressFromView(this); } private void CalculatePairwise(string groupName) { if (!VerifyDataLength(false)) return; MainFormManager.GetMainForm().AddOperationProgressToView(this, "Calculating..."); Task.Factory.StartNew(() => CalculatePairwiseAsync(groupName)); } private void CalculatePairwiseAsync(string groupName) { CalculatePairwiseTest(groupName); CalculatePairwiseTestDetails(groupName); MainFormManager.GetMainForm().RemoveOperationProgressFromView(this); } private void TestAllGroups() { double pval = KruskalWallis.Test(data); pValTextBox.Text = pval.ToString(); if (pval < significanceLevel) { this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; })); } else { this.Invoke(new Action(() => { groupCompLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; })); } } private void CalculateNormality() { double val; List res = new List(); for (int i = 0; i < data.Length; i++) { alglib.jarqueberatest(data[i], data[i].Length, out val); res.Add(val); } // p-value is below significance level and thus the null hypothesis (data is normally distributed) is rejected. if (res.Any(x => x < significanceLevel)) { this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; })); } else { this.Invoke(new Action(() => { normalityLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; })); } } private void CalculateNormalityDetails() { DoubleMatrix pValsMatrix = new DoubleMatrix(1, stringConvertibleMatrixView.Content.Columns); pValsMatrix.ColumnNames = stringConvertibleMatrixView.Content.ColumnNames; pValsMatrix.RowNames = new string[] { "p-Value" }; double val; for (int i = 0; i < data.Length; i++) { alglib.jarqueberatest(data[i], data[i].Length, out val); pValsMatrix[0, i] = val; } this.Invoke(new Action(() => { normalityStringConvertibleMatrixView.Content = pValsMatrix; })); } private void CalculatePairwiseTest(string groupName) { int colIndex = 0; IEnumerable columnNames = null; this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; })); foreach (string col in columnNames) { if (col == groupName) { break; } colIndex++; } double[][] newData = FilterDataForPairwiseTest(colIndex); double mwuBothtails; double mwuLefttail; double mwuRighttail; int cnt = 0; for (int i = 0; i < newData.Length; i++) { alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, newData[i], newData[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail); if (mwuBothtails > significanceLevel) { cnt++; } } double ratio = ((double)cnt) / (data.Length - 1) * 100.0; equalDistsTextBox.Text = ratio.ToString() + " %"; if (cnt == 0) { this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Analysis.Statistics.Resources.Default; })); } else { this.Invoke(new Action(() => { pairwiseLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Warning; })); } } private double[][] FilterDataForPairwiseTest(int columnToRemove) { double[][] newData = new double[data.Length - 1][]; int i = 0; int l = 0; while (i < data.Length) { if (i != columnToRemove) { double[] row = new double[data[i].Length - 1]; newData[l] = row; int j = 0, k = 0; while (j < row.Length) { if (i != columnToRemove) { newData[l][j] = data[i][k]; j++; k++; } else { k++; } } i++; l++; } else { i++; } } return newData; } private void CalculatePairwiseTestDetails(string groupName) { int colIndex = 0; IEnumerable columnNames = null; this.Invoke(new Action(() => { columnNames = stringConvertibleMatrixView.Content.ColumnNames; })); foreach (string col in columnNames) { if (col == groupName) { break; } colIndex++; } double[][] newData = FilterDataForPairwiseTest(colIndex); columnNames = columnNames.Where(x => x != groupName).ToList(); var rowNames = new string[] { "p-Value of Mann-Whitney U", "Adjusted p-Value of Mann-Whitney U", "p-Value of T-Test", "Adjusted p-Value of T-Test", "Necessary Sample Size for T-Test", "Cohen's d", "Hedges' g" }; DoubleMatrix pValsMatrix = new DoubleMatrix(rowNames.Length, columnNames.Count()); pValsMatrix.ColumnNames = columnNames; pValsMatrix.RowNames = rowNames; double mwuBothtails; double mwuLefttail; double mwuRighttail; double tTestLefttail; double[] mwuPValues = new double[newData.Length]; double[] tTestPValues = new double[newData.Length]; bool[] decision = null; double[] adjustedMwuPValues = null; double[] adjustedTtestPValues = null; for (int i = 0; i < newData.Length; i++) { if (i != colIndex) { alglib.mannwhitneyutest(data[colIndex], data[colIndex].Length, newData[i], newData[i].Length, out mwuBothtails, out mwuLefttail, out mwuRighttail); tTestLefttail = TTest.Test(data[colIndex], newData[i]); mwuPValues[i] = mwuBothtails; tTestPValues[i] = tTestLefttail; } } adjustedMwuPValues = BonferroniHolm.Calculate(significanceLevel, mwuPValues, out decision); adjustedTtestPValues = BonferroniHolm.Calculate(significanceLevel, tTestPValues, out decision); for (int i = 0; i < newData.Length; i++) { if (i != colIndex) { pValsMatrix[0, i] = mwuPValues[i]; pValsMatrix[1, i] = adjustedMwuPValues[i]; pValsMatrix[2, i] = tTestPValues[i]; pValsMatrix[3, i] = adjustedTtestPValues[i]; pValsMatrix[4, i] = TTest.GetOptimalSampleSize(data[colIndex], newData[i]); pValsMatrix[5, i] = SampleSizeDetermination.CalculateCohensD(data[colIndex], newData[i]); pValsMatrix[6, i] = SampleSizeDetermination.CalculateHedgesG(data[colIndex], newData[i]); } } this.Invoke(new Action(() => { pairwiseStringConvertibleMatrixView.Content = pValsMatrix; })); } private void openBoxPlotToolStripMenuItem_Click(object sender, EventArgs e) { RunCollectionBoxPlotView boxplotView = new RunCollectionBoxPlotView(); boxplotView.Content = Content; // TODO: enable as soon as we move to HeuristicLab.Optimization.Views // boxplotView.xAxisComboBox.SelectedItem = xAxisComboBox.SelectedItem; // boxplotView.yAxisComboBox.SelectedItem = yAxisComboBox.SelectedItem; boxplotView.Show(); } private void groupCompComboBox_SelectedValueChanged(object sender, EventArgs e) { string curItem = (string)groupCompComboBox.SelectedItem; CalculatePairwise(curItem); } } }