Index: /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.Designer.cs
===================================================================
--- /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.Designer.cs	(revision 15940)
+++ /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.Designer.cs	(revision 15940)
@@ -0,0 +1,225 @@
+﻿namespace HeuristicLab.Optimization.Views.RunCollectionViews {
+  partial class RunCollectionResultAggregationView {
+    /// <summary> 
+    /// Required designer variable.
+    /// </summary>
+    private System.ComponentModel.IContainer components = null;
+
+    /// <summary> 
+    /// Clean up any resources being used.
+    /// </summary>
+    /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
+    protected override void Dispose(bool disposing) {
+      if (disposing && (components != null)) {
+        components.Dispose();
+      }
+      base.Dispose(disposing);
+    }
+
+    #region Component Designer generated code
+
+    /// <summary> 
+    /// Required method for Designer support - do not modify 
+    /// the contents of this method with the code editor.
+    /// </summary>
+    private void InitializeComponent() {
+      this.aggregatedResultsMatrixView = new HeuristicLab.Data.Views.StringConvertibleMatrixView();
+      this.resultsGroupBox = new System.Windows.Forms.GroupBox();
+      this.resultsCheckedList = new System.Windows.Forms.CheckedListBox();
+      this.groupBox1 = new System.Windows.Forms.GroupBox();
+      this.statisticsCheckedList = new System.Windows.Forms.CheckedListBox();
+      this.groupingGroupBox = new System.Windows.Forms.GroupBox();
+      this.secondCriterionComboBox = new System.Windows.Forms.ComboBox();
+      this.label2 = new System.Windows.Forms.Label();
+      this.firstCriterionComboBox = new System.Windows.Forms.ComboBox();
+      this.label1 = new System.Windows.Forms.Label();
+      this.panel1 = new System.Windows.Forms.Panel();
+      this.orderByStatisticCheckbox = new System.Windows.Forms.CheckBox();
+      this.transposeMatrixCheckBox = new System.Windows.Forms.CheckBox();
+      this.resultsGroupBox.SuspendLayout();
+      this.groupBox1.SuspendLayout();
+      this.groupingGroupBox.SuspendLayout();
+      this.panel1.SuspendLayout();
+      this.SuspendLayout();
+      // 
+      // aggregatedResultsMatrixView
+      // 
+      this.aggregatedResultsMatrixView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left) 
+            | System.Windows.Forms.AnchorStyles.Right)));
+      this.aggregatedResultsMatrixView.Caption = "StringConvertibleMatrix View";
+      this.aggregatedResultsMatrixView.Content = null;
+      this.aggregatedResultsMatrixView.Location = new System.Drawing.Point(368, 3);
+      this.aggregatedResultsMatrixView.Name = "aggregatedResultsMatrixView";
+      this.aggregatedResultsMatrixView.ReadOnly = true;
+      this.aggregatedResultsMatrixView.ShowRowsAndColumnsTextBox = false;
+      this.aggregatedResultsMatrixView.ShowStatisticalInformation = true;
+      this.aggregatedResultsMatrixView.Size = new System.Drawing.Size(665, 643);
+      this.aggregatedResultsMatrixView.TabIndex = 1;
+      // 
+      // resultsGroupBox
+      // 
+      this.resultsGroupBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left)));
+      this.resultsGroupBox.AutoSize = true;
+      this.resultsGroupBox.Controls.Add(this.resultsCheckedList);
+      this.resultsGroupBox.Location = new System.Drawing.Point(4, 245);
+      this.resultsGroupBox.Name = "resultsGroupBox";
+      this.resultsGroupBox.Size = new System.Drawing.Size(349, 349);
+      this.resultsGroupBox.TabIndex = 4;
+      this.resultsGroupBox.TabStop = false;
+      this.resultsGroupBox.Text = "Results";
+      // 
+      // resultsCheckedList
+      // 
+      this.resultsCheckedList.CheckOnClick = true;
+      this.resultsCheckedList.Dock = System.Windows.Forms.DockStyle.Fill;
+      this.resultsCheckedList.FormattingEnabled = true;
+      this.resultsCheckedList.Location = new System.Drawing.Point(3, 16);
+      this.resultsCheckedList.Name = "resultsCheckedList";
+      this.resultsCheckedList.Size = new System.Drawing.Size(343, 330);
+      this.resultsCheckedList.TabIndex = 0;
+      this.resultsCheckedList.ItemCheck += new System.Windows.Forms.ItemCheckEventHandler(this.resultsCheckedList_ItemCheck);
+      // 
+      // groupBox1
+      // 
+      this.groupBox1.Controls.Add(this.statisticsCheckedList);
+      this.groupBox1.Location = new System.Drawing.Point(4, 85);
+      this.groupBox1.Name = "groupBox1";
+      this.groupBox1.Size = new System.Drawing.Size(349, 154);
+      this.groupBox1.TabIndex = 5;
+      this.groupBox1.TabStop = false;
+      this.groupBox1.Text = "Show";
+      // 
+      // statisticsCheckedList
+      // 
+      this.statisticsCheckedList.CheckOnClick = true;
+      this.statisticsCheckedList.Dock = System.Windows.Forms.DockStyle.Fill;
+      this.statisticsCheckedList.FormattingEnabled = true;
+      this.statisticsCheckedList.Location = new System.Drawing.Point(3, 16);
+      this.statisticsCheckedList.Name = "statisticsCheckedList";
+      this.statisticsCheckedList.Size = new System.Drawing.Size(343, 135);
+      this.statisticsCheckedList.TabIndex = 1;
+      this.statisticsCheckedList.ItemCheck += new System.Windows.Forms.ItemCheckEventHandler(this.statisticsToCalculateCheckBox_ItemCheck);
+      // 
+      // groupingGroupBox
+      // 
+      this.groupingGroupBox.Controls.Add(this.secondCriterionComboBox);
+      this.groupingGroupBox.Controls.Add(this.label2);
+      this.groupingGroupBox.Controls.Add(this.firstCriterionComboBox);
+      this.groupingGroupBox.Controls.Add(this.label1);
+      this.groupingGroupBox.Location = new System.Drawing.Point(4, 3);
+      this.groupingGroupBox.Name = "groupingGroupBox";
+      this.groupingGroupBox.Size = new System.Drawing.Size(349, 76);
+      this.groupingGroupBox.TabIndex = 6;
+      this.groupingGroupBox.TabStop = false;
+      this.groupingGroupBox.Text = "Group by";
+      // 
+      // secondCriterionComboBox
+      // 
+      this.secondCriterionComboBox.FormattingEnabled = true;
+      this.secondCriterionComboBox.Location = new System.Drawing.Point(98, 44);
+      this.secondCriterionComboBox.Name = "secondCriterionComboBox";
+      this.secondCriterionComboBox.Size = new System.Drawing.Size(245, 21);
+      this.secondCriterionComboBox.TabIndex = 3;
+      this.secondCriterionComboBox.SelectedIndexChanged += new System.EventHandler(this.criterionComboBox_SelectedIndexChanged);
+      // 
+      // label2
+      // 
+      this.label2.AutoSize = true;
+      this.label2.Location = new System.Drawing.Point(3, 47);
+      this.label2.Name = "label2";
+      this.label2.Size = new System.Drawing.Size(84, 13);
+      this.label2.TabIndex = 2;
+      this.label2.Text = "Second criterion";
+      // 
+      // firstCriterionComboBox
+      // 
+      this.firstCriterionComboBox.FormattingEnabled = true;
+      this.firstCriterionComboBox.Location = new System.Drawing.Point(98, 17);
+      this.firstCriterionComboBox.Name = "firstCriterionComboBox";
+      this.firstCriterionComboBox.Size = new System.Drawing.Size(245, 21);
+      this.firstCriterionComboBox.TabIndex = 1;
+      this.firstCriterionComboBox.SelectedIndexChanged += new System.EventHandler(this.criterionComboBox_SelectedIndexChanged);
+      // 
+      // label1
+      // 
+      this.label1.AutoSize = true;
+      this.label1.Location = new System.Drawing.Point(3, 20);
+      this.label1.Name = "label1";
+      this.label1.Size = new System.Drawing.Size(66, 13);
+      this.label1.TabIndex = 0;
+      this.label1.Text = "First criterion";
+      // 
+      // panel1
+      // 
+      this.panel1.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left)));
+      this.panel1.Controls.Add(this.orderByStatisticCheckbox);
+      this.panel1.Controls.Add(this.transposeMatrixCheckBox);
+      this.panel1.Controls.Add(this.groupingGroupBox);
+      this.panel1.Controls.Add(this.groupBox1);
+      this.panel1.Controls.Add(this.resultsGroupBox);
+      this.panel1.Location = new System.Drawing.Point(3, 3);
+      this.panel1.Name = "panel1";
+      this.panel1.Size = new System.Drawing.Size(359, 643);
+      this.panel1.TabIndex = 0;
+      // 
+      // orderByStatisticCheckbox
+      // 
+      this.orderByStatisticCheckbox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left)));
+      this.orderByStatisticCheckbox.AutoSize = true;
+      this.orderByStatisticCheckbox.Location = new System.Drawing.Point(4, 600);
+      this.orderByStatisticCheckbox.Name = "orderByStatisticCheckbox";
+      this.orderByStatisticCheckbox.Size = new System.Drawing.Size(129, 17);
+      this.orderByStatisticCheckbox.TabIndex = 7;
+      this.orderByStatisticCheckbox.Text = "Group similar statistics";
+      this.orderByStatisticCheckbox.UseVisualStyleBackColor = true;
+      this.orderByStatisticCheckbox.CheckedChanged += new System.EventHandler(this.orderByStatisticCheckbox_CheckedChanged);
+      // 
+      // transposeMatrixCheckBox
+      // 
+      this.transposeMatrixCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left)));
+      this.transposeMatrixCheckBox.AutoSize = true;
+      this.transposeMatrixCheckBox.Location = new System.Drawing.Point(4, 623);
+      this.transposeMatrixCheckBox.Name = "transposeMatrixCheckBox";
+      this.transposeMatrixCheckBox.Size = new System.Drawing.Size(106, 17);
+      this.transposeMatrixCheckBox.TabIndex = 7;
+      this.transposeMatrixCheckBox.Text = "Transpose matrix";
+      this.transposeMatrixCheckBox.UseVisualStyleBackColor = true;
+      this.transposeMatrixCheckBox.CheckedChanged += new System.EventHandler(this.transposeMatrixCheckBox_CheckedChanged);
+      // 
+      // RunCollectionResultAggregationView
+      // 
+      this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
+      this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
+      this.Controls.Add(this.aggregatedResultsMatrixView);
+      this.Controls.Add(this.panel1);
+      this.Name = "RunCollectionResultAggregationView";
+      this.Size = new System.Drawing.Size(1036, 649);
+      this.resultsGroupBox.ResumeLayout(false);
+      this.groupBox1.ResumeLayout(false);
+      this.groupingGroupBox.ResumeLayout(false);
+      this.groupingGroupBox.PerformLayout();
+      this.panel1.ResumeLayout(false);
+      this.panel1.PerformLayout();
+      this.ResumeLayout(false);
+
+    }
+
+    #endregion
+    private Data.Views.StringConvertibleMatrixView aggregatedResultsMatrixView;
+    private System.Windows.Forms.GroupBox resultsGroupBox;
+    private System.Windows.Forms.CheckedListBox resultsCheckedList;
+    private System.Windows.Forms.GroupBox groupBox1;
+    private System.Windows.Forms.CheckedListBox statisticsCheckedList;
+    private System.Windows.Forms.GroupBox groupingGroupBox;
+    private System.Windows.Forms.ComboBox secondCriterionComboBox;
+    private System.Windows.Forms.Label label2;
+    private System.Windows.Forms.ComboBox firstCriterionComboBox;
+    private System.Windows.Forms.Label label1;
+    private System.Windows.Forms.Panel panel1;
+    private System.Windows.Forms.CheckBox orderByStatisticCheckbox;
+    private System.Windows.Forms.CheckBox transposeMatrixCheckBox;
+  }
+}
Index: /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.cs
===================================================================
--- /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.cs	(revision 15940)
+++ /trunk/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionResultAggregationView.cs	(revision 15940)
@@ -0,0 +1,245 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Windows.Forms;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.MainForm;
+using HeuristicLab.MainForm.WindowsForms;
+
+namespace HeuristicLab.Optimization.Views.RunCollectionViews {
+  [View("Result Aggregation")]
+  [Content(typeof(RunCollection), false)]
+  public partial class RunCollectionResultAggregationView : AsynchronousContentView {
+    private enum AggregationMethod { Minimum, Maximum, Average, FirstQuartile, Median, ThirdQuartile, IQR, StdDev }
+
+    private readonly Dictionary<AggregationMethod, Func<IEnumerable<double>, double>> aggregator = new Dictionary<AggregationMethod, Func<IEnumerable<double>, double>> {
+      { AggregationMethod.Minimum, x => x.Min() },
+      { AggregationMethod.Maximum, x => x.Max() },
+      { AggregationMethod.Average, x => x.Average() },
+      { AggregationMethod.FirstQuartile, x => x.Quantile(0.25) },
+      { AggregationMethod.Median, x => x.Median() },
+      { AggregationMethod.ThirdQuartile, x => x.Quantile(0.75) },
+      { AggregationMethod.IQR, x => x.Quantile(0.75) - x.Quantile(0.25) },
+      { AggregationMethod.StdDev, x => x.StandardDeviation() },
+    };
+
+    public RunCollectionResultAggregationView() {
+      InitializeComponent();
+    }
+
+    public new RunCollection Content {
+      get { return (RunCollection)base.Content; }
+      set { base.Content = value; }
+    }
+
+    protected override void OnContentChanged() {
+      var results = GetCompatibleResults(Content);
+
+      statisticsCheckedList.Items.Clear();
+      resultsCheckedList.Items.Clear();
+
+      foreach (var name in Enum.GetNames(typeof(AggregationMethod))) {
+        statisticsCheckedList.Items.Add(name, false);
+      }
+
+      foreach (var name in GetCompatibleResults(Content).OrderBy(x => x))
+        resultsCheckedList.Items.Add(name, false);
+
+      firstCriterionComboBox.Items.Clear();
+      secondCriterionComboBox.Items.Clear();
+
+      firstCriterionComboBox.Items.AddRange(Content.ParameterNames.OrderBy(x => x).ToArray());
+      secondCriterionComboBox.Items.AddRange(Content.ParameterNames.OrderBy(x => x).ToArray());
+
+      base.OnContentChanged();
+    }
+
+    private static IEnumerable<string> GetCompatibleResults(RunCollection runs) {
+      var results = new HashSet<string>(runs.First().Results.Where(x => IsCompatible(x.Value)).Select(x => x.Key));
+
+      foreach (var run in runs.Skip(1)) {
+        results.IntersectWith(run.Results.Keys);
+      }
+
+      return results;
+    }
+
+    // return only results of a numeric type
+    private static bool IsCompatible(IItem item) {
+      return item is IntValue || item is DoubleValue || item is PercentValue || item is TimeSpanValue;
+    }
+
+    private void AggregateResults() {
+      if (!(statisticsCheckedList.CheckedItems.Count > 0 && resultsCheckedList.CheckedItems.Count > 0 && !string.IsNullOrEmpty(firstCriterionComboBox.Text)))
+        return;
+
+      var groups = new Dictionary<string, IEnumerable<IRun>>();
+      var first = Content.GroupBy(run => Content.GetValue(run, firstCriterionComboBox.Text).ToString());
+
+      if (!string.IsNullOrEmpty(secondCriterionComboBox.Text) && secondCriterionComboBox.Text != firstCriterionComboBox.Text) {
+        foreach (var f in first) {
+          foreach (var g in f.GroupBy(run => Content.GetValue(run, secondCriterionComboBox.Text).ToString())) {
+            var key = f.Key + " " + g.Key;
+            groups.Add(key, g);
+          }
+        }
+      } else {
+        groups = first.ToDictionary(x => x.Key, x => x.AsEnumerable());
+      }
+
+      var aggregatedDict = new Dictionary<string, List<Tuple<string, double>>>();
+
+      foreach (var result in resultsCheckedList.CheckedItems) {
+        var resultName = result.ToString();
+        foreach (var group in groups) {
+          var values = GetValues(group.Value, resultName);
+
+          var key = group.Key;
+          if (!aggregatedDict.ContainsKey(key)) {
+            aggregatedDict[key] = new List<Tuple<string, double>>();
+          }
+
+          foreach (var i in statisticsCheckedList.CheckedIndices) {
+            var name = resultName + " " + (AggregationMethod)i;
+            aggregatedDict[key].Add(Tuple.Create(name, aggregator[(AggregationMethod)i](values)));
+          }
+        }
+      }
+      var groupNames = aggregatedDict.Keys.ToList();
+      var resultNames = aggregatedDict.First().Value.Select(x => x.Item1).ToList();
+
+      if (!aggregatedDict.Values.Skip(1).All(x => resultNames.SequenceEqual(x.Select(y => y.Item1)))) {
+        throw new Exception("Inconsistent results across groups.");
+      }
+
+      var rows = resultNames.Count;
+      var cols = groups.Count;
+
+      var matrix = new DoubleMatrix(rows, cols);
+
+      var col = 0;
+      foreach (var groupName in groupNames) {
+        var row = 0;
+        foreach (var resultPair in aggregatedDict[groupName]) {
+          matrix[row, col] = resultPair.Item2;
+          ++row;
+        }
+        ++col;
+      }
+      matrix.RowNames = resultNames;
+      matrix.ColumnNames = groupNames;
+
+      aggregatedResultsMatrixView.Content = transposeMatrixCheckBox.Checked ? Transpose(matrix) : matrix;
+    }
+
+    private IEnumerable<double> GetValues(IEnumerable<IRun> runs, string resultName) {
+      var values = new List<double>();
+
+      foreach (var run in runs) {
+        var value = Content.GetValue(run, resultName);
+        var doubleValue = value as DoubleValue;
+        var intValue = value as IntValue;
+        var percentValue = value as PercentValue;
+        var timeSpanValue = value as TimeSpanValue;
+
+        if (doubleValue != null) {
+          values.Add(doubleValue.Value);
+        } else if (intValue != null) {
+          values.Add(intValue.Value);
+        } else if (percentValue != null) {
+          values.Add(percentValue.Value);
+        } else if (timeSpanValue != null) {
+          values.Add(timeSpanValue.Value.TotalSeconds);
+        }
+      }
+      return values;
+    }
+
+    private static DoubleMatrix Transpose(DoubleMatrix matrix) {
+      var transposed = new DoubleMatrix(matrix.Columns, matrix.Rows) {
+        RowNames = matrix.ColumnNames,
+        ColumnNames = matrix.RowNames
+      };
+
+      for (int i = 0; i < matrix.Rows; ++i) {
+        for (int j = 0; j < matrix.Columns; ++j) {
+          transposed[j, i] = matrix[i, j];
+        }
+      }
+
+      return transposed;
+    }
+
+    #region events 
+    private void aggregateByComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      AggregateResults();
+    }
+
+    private void resultsCheckedList_ItemCheck(object sender, ItemCheckEventArgs e) {
+      BeginInvoke((MethodInvoker)AggregateResults); // delayed execution
+    }
+
+    private void statisticsToCalculateCheckBox_ItemCheck(object sender, ItemCheckEventArgs e) {
+      BeginInvoke((MethodInvoker)AggregateResults); // delayed execution
+    }
+
+    private void criterionComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      AggregateResults();
+    }
+
+    private void transposeMatrixCheckBox_CheckedChanged(object sender, EventArgs e) {
+      if (aggregatedResultsMatrixView.Content == null)
+        return;
+
+      var matrix = (DoubleMatrix)aggregatedResultsMatrixView.Content;
+      aggregatedResultsMatrixView.Content = Transpose(matrix);
+    }
+
+    private void orderByStatisticCheckbox_CheckedChanged(object sender, EventArgs e) {
+      if (!orderByStatisticCheckbox.Checked) {
+        AggregateResults();
+        return;
+      }
+
+      // we reorder the matrix rows (or columns, if transposed) so that every n_th rows (column) become successive
+      // the number of rows (columns, if transposed) in the matrix is a multiple of n
+      var matrix = (DoubleMatrix)aggregatedResultsMatrixView.Content;
+      var rowNames = matrix.RowNames.ToList();
+      var columnNames = matrix.ColumnNames.ToList();
+
+      var ordered = new DoubleMatrix(matrix.Rows, matrix.Columns);
+      var orderedNames = new List<string>();
+
+      var transposed = transposeMatrixCheckBox.Checked;
+      var count = statisticsCheckedList.CheckedItems.Count;
+      var outerCount = transposed ? matrix.Columns : matrix.Rows;
+      var innerCount = transposed ? matrix.Rows : matrix.Columns;
+
+      for (int i = 0; i < count; ++i) {
+        for (int j = i; j < outerCount; j += count) {
+          var c = orderedNames.Count;
+          for (int k = 0; k < innerCount; ++k) {
+            if (transposed)
+              ordered[k, c] = matrix[k, j];
+            else
+              ordered[c, k] = matrix[j, k];
+          }
+          orderedNames.Add(transposed ? columnNames[j] : rowNames[j]);
+        }
+      }
+
+      if (transposed) {
+        ordered.RowNames = matrix.RowNames;
+        ordered.ColumnNames = orderedNames;
+      } else {
+        ordered.ColumnNames = matrix.ColumnNames;
+        ordered.RowNames = orderedNames;
+      }
+
+      aggregatedResultsMatrixView.Content = ordered;
+    }
+    #endregion
+  }
+}
