#region License Information
/* HeuristicLab
* Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
*
* This file is part of HeuristicLab.
*
* HeuristicLab is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* HeuristicLab is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HeuristicLab. If not, see .
*/
#endregion
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Linq;
using System.Windows.Forms;
using HeuristicLab.Analysis;
using HeuristicLab.Collections;
using HeuristicLab.Common;
using HeuristicLab.Core.Views;
using HeuristicLab.MainForm;
using System.Windows.Forms.DataVisualization.Charting;
using System.Text;
using HeuristicLab.Data;
using System.Drawing;
using HeuristicLab.Core;
namespace HeuristicLab.Optimization.Views {
[View("Parameter Analysis")]
[Content(typeof(RunCollection), false)]
public sealed partial class RunCollectionParameterAnalysisView : ItemView {
#region Colors
private static readonly Color[] colors = new[] {
Color.FromArgb(0x40, 0x6A, 0xB7),
Color.FromArgb(0xB1, 0x6D, 0x01),
Color.FromArgb(0x4E, 0x8A, 0x06),
Color.FromArgb(0x75, 0x50, 0x7B),
Color.FromArgb(0x72, 0x9F, 0xCF),
Color.FromArgb(0xA4, 0x00, 0x00),
Color.FromArgb(0xAD, 0x7F, 0xA8),
Color.FromArgb(0x29, 0x50, 0xCF),
Color.FromArgb(0x90, 0xB0, 0x60),
Color.FromArgb(0xF5, 0x89, 0x30),
Color.FromArgb(0x55, 0x57, 0x53),
Color.FromArgb(0xEF, 0x59, 0x59),
Color.FromArgb(0xED, 0xD4, 0x30),
Color.FromArgb(0x63, 0xC2, 0x16),
};
#endregion
private bool suppressUpdates = false;
private int stepSize = 1000;
private Dictionary>> runData;
private Dictionary paramInfos;
public new RunCollection Content {
get { return (RunCollection)base.Content; }
set { base.Content = value; }
}
public RunCollectionParameterAnalysisView() {
InitializeComponent();
chart.CustomizeAllChartAreas();
stepSizeTextBox.Text = stepSize.ToString();
errorProvider.SetIconAlignment(stepSizeTextBox, ErrorIconAlignment.MiddleLeft);
errorProvider.SetIconPadding(stepSizeTextBox, 2);
}
#region Content Events
protected override void DeregisterContentEvents() {
Content.ItemsAdded -= Content_ItemsAdded;
Content.ItemsRemoved -= Content_ItemsRemoved;
Content.CollectionReset -= Content_CollectionReset;
Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
DeregisterRunEvents(Content);
base.DeregisterContentEvents();
}
protected override void RegisterContentEvents() {
base.RegisterContentEvents();
Content.ItemsAdded += Content_ItemsAdded;
Content.ItemsRemoved += Content_ItemsRemoved;
Content.CollectionReset += Content_CollectionReset;
Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
RegisterRunEvents(Content);
}
private void DeregisterRunEvents(IEnumerable runs) {
foreach (var run in runs)
run.PropertyChanged -= Run_PropertyChanged;
}
private void RegisterRunEvents(IEnumerable runs) {
foreach (var run in runs)
run.PropertyChanged += Run_PropertyChanged;
}
private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs e) {
RegisterRunEvents(e.Items);
}
private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs e) {
DeregisterRunEvents(e.Items);
}
private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs e) {
DeregisterRunEvents(e.OldItems);
RegisterRunEvents(e.Items);
}
private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
if (InvokeRequired)
Invoke((Action)Content_UpdateOfRunsInProgressChanged, sender, e);
else {
suppressUpdates = Content.UpdateOfRunsInProgress;
if (suppressUpdates) return;
CollectRunData();
AnalyzeParameters();
UpdateChart();
}
}
private void Run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
if (suppressUpdates) return;
if (InvokeRequired)
Invoke((Action)Run_PropertyChanged, sender, e);
else {
CollectRunData();
AnalyzeParameters();
UpdateChart();
}
}
#endregion
protected override void OnContentChanged() {
base.OnContentChanged();
CollectRunData();
AnalyzeParameters();
UpdateChart();
}
protected override void SetEnabledStateOfControls() {
base.SetEnabledStateOfControls();
parametersGroupBox.Enabled = Content != null;
groupsGroupBox.Enabled = Content != null;
stepSizeTextBox.Enabled = Content != null;
logScalingCheckBox.Enabled = Content != null;
dataRowsGroupBox.Enabled = Content != null;
}
private void AnalyzeParameters() {
paramInfos = new Dictionary();
if (Content == null) return;
foreach (var run in runData.Keys) {
foreach (var param in run.Parameters) {
ParameterInfo info;
if (!paramInfos.TryGetValue(param.Key, out info))
paramInfos.Add(param.Key, new ParameterInfo(param.Key, param.Value.ToString(), run));
else
info.AddValue(param.Value.ToString(), run);
}
}
// remove irrelevant parameters
if (paramInfos.ContainsKey("Seed"))
paramInfos.Remove("Seed");
// remove all parameters which only have a single value
var singles = new List();
foreach (var paramInfo in paramInfos.Values) {
if (paramInfo.Values.Count == 1)
singles.Add(paramInfo.Name);
}
foreach (var single in singles)
paramInfos.Remove(single);
// set color of parameter values
int i = 0;
foreach (var valueInfo in paramInfos.Values.SelectMany(x => x.Values.Values)) {
valueInfo.Color = colors[i];
i = (i + 1) % colors.Length;
}
// populate parametersTreeView
parametersTreeView.Nodes.Clear();
var paramsRoot = new TreeNode("All Runs (" + runData.Keys.Count + ")");
paramsRoot.Tag = runData.Keys.ToList();
foreach (var paramInfo in paramInfos.Values.OrderBy(x => x.Name)) {
var node = new TreeNode(paramInfo.Name + " (" + paramInfo.RunCount + ")");
foreach (var value in paramInfo.Values) {
var child = new TreeNode(value.Key + " (" + value.Value.RunCount + ")");
child.Tag = value.Value;
node.Nodes.Add(child);
}
node.Tag = paramInfo;
paramsRoot.Nodes.Add(node);
}
parametersTreeView.Nodes.Add(paramsRoot);
paramsRoot.Expand();
paramsRoot.Checked = true;
parametersTreeView.SelectedNode = paramsRoot;
// populate groupsTreeView
groupsTreeView.Nodes.Clear();
var groupsRoot = new TreeNode("All Runs (" + runData.Keys.Count + ")");
groupsRoot.Tag = runData.Keys.ToList();
groupsTreeView.Nodes.Add(groupsRoot);
groupsRoot.Expand();
groupsTreeView.SelectedNode = groupsRoot;
}
private void CollectRunData() {
runData = new Dictionary>>();
if (Content == null) return;
foreach (var run in Content) {
try {
IList> values = (run.Results["QualityPerEvaluations"] as IndexedDataTable).Rows["First-hit Graph"].Values;
var bestKnown = (run.Results["BestKnownQuality"] as DoubleValue).Value;
var data = new List>();
int i = 0;
double qual = double.NaN;
foreach (var val in values) {
while (i * stepSize < val.Item1) {
if (!double.IsNaN(qual)) data.Add(Tuple.Create(i * stepSize, qual));
i++;
}
var diff = Math.Abs(bestKnown - val.Item2);
qual = bestKnown == 0 ? diff : diff / bestKnown;
}
runData.Add(run, data);
}
catch {
}
}
}
private void UpdateChart() {
if (suppressUpdates) return;
chart.Series.Clear();
chart.Legends[0].CustomItems.Clear();
chart.Titles[0].Text = string.Empty;
var checkedNodes = TraverseTreeNodes(parametersTreeView.Nodes)
.Where(x => x.Checked)
.Concat(
TraverseTreeNodes(groupsTreeView.Nodes)
.Where(x => x.Checked)
);
foreach (var node in checkedNodes) {
if (node.Parent == null) {
var series = BuildSeries(node.Tag as IEnumerable, colors[0]);
chart.Titles[0].Text = node.Text;
foreach (var s in series)
chart.Series.Add(s);
var legendItem = new LegendItem();
var legendItemInfo = new LegendItemInfo(colors[0], series);
legendItem.Color = legendItemInfo.Color;
legendItem.BorderColor = Color.Transparent;
legendItem.Name = node.Text;
legendItem.Tag = legendItemInfo;
chart.Legends[0].CustomItems.Add(legendItem);
} else if (node.Tag is ParameterInfo) {
var paramInfo = node.Tag as ParameterInfo;
chart.Titles[0].Text = paramInfo.Name + " (" + paramInfo.RunCount + ")";
foreach (var value in paramInfo.Values) {
var series = BuildSeries(value.Value.Runs, value.Value.Color);
foreach (var s in series)
chart.Series.Add(s);
var legendItem = new LegendItem();
var legendItemInfo = new LegendItemInfo(value.Value.Color, series);
legendItem.Color = legendItemInfo.Color;
legendItem.BorderColor = Color.Transparent;
legendItem.Name = value.Key + " (" + value.Value.RunCount + ")";
legendItem.Tag = legendItemInfo;
chart.Legends[0].CustomItems.Add(legendItem);
}
} else if (node.Tag is ParameterValueInfo) {
var valueInfo = node.Tag as ParameterValueInfo;
var series = BuildSeries(valueInfo.Runs, valueInfo.Color);
chart.Titles[0].Text = node.Parent.Text;
foreach (var s in series)
chart.Series.Add(s);
var legendItem = new LegendItem();
var legendItemInfo = new LegendItemInfo(valueInfo.Color, series);
legendItem.Color = legendItemInfo.Color;
legendItem.BorderColor = Color.Transparent;
legendItem.Name = node.Text;
legendItem.Tag = legendItemInfo;
chart.Legends[0].CustomItems.Add(legendItem);
} else if (node.Tag is GroupInfo) {
var groupInfo = node.Tag as GroupInfo;
if (groupInfo.IsParameter) {
chart.Titles[0].Text = groupInfo.Text + " (" + groupInfo.Runs.Count + ")";
foreach (TreeNode child in node.Nodes) {
var childInfo = child.Tag as GroupInfo;
var series = BuildSeries(childInfo.Runs, childInfo.Color);
foreach (var s in series)
chart.Series.Add(s);
var legendItem = new LegendItem();
var legendItemInfo = new LegendItemInfo(childInfo.Color, series);
legendItem.Color = legendItemInfo.Color;
legendItem.BorderColor = Color.Transparent;
legendItem.Name = childInfo.Text + " (" + childInfo.Runs.Count + ")";
legendItem.Tag = legendItemInfo;
chart.Legends[0].CustomItems.Add(legendItem);
}
} else {
var parentInfo = node.Parent.Tag as GroupInfo;
chart.Titles[0].Text = parentInfo.Text + " (" + parentInfo.Runs.Count + ")";
var series = BuildSeries(groupInfo.Runs, groupInfo.Color);
foreach (var s in series)
chart.Series.Add(s);
var legendItem = new LegendItem();
var legendItemInfo = new LegendItemInfo(groupInfo.Color, series);
legendItem.Color = legendItemInfo.Color;
legendItem.BorderColor = Color.Transparent;
legendItem.Name = groupInfo.Text + " (" + groupInfo.Runs.Count + ")";
legendItem.Tag = legendItemInfo;
chart.Legends[0].CustomItems.Add(legendItem);
}
}
}
}
private void UpdateSeriesVisibility() {
foreach (var legendItem in chart.Legends[0].CustomItems) {
var legendItemInfo = legendItem.Tag as LegendItemInfo;
foreach (var s in legendItemInfo.Series) {
if (legendItemInfo.SeriesVisible) {
var seriesInfo = s.Tag as SeriesInfo;
switch (seriesInfo.Type) {
case SeriesTypes.MinMax:
s.Color = minMaxCheckBox.Checked ? seriesInfo.Color : Color.Transparent;
break;
case SeriesTypes.Quartiles:
s.Color = quartilesCheckBox.Checked ? seriesInfo.Color : Color.Transparent;
break;
case SeriesTypes.Median:
s.Color = medianCheckBox.Checked ? seriesInfo.Color : Color.Transparent;
break;
case SeriesTypes.Average:
s.Color = averageCheckBox.Checked ? seriesInfo.Color : Color.Transparent;
break;
}
} else {
s.Color = Color.Transparent;
}
}
}
}
private IEnumerable BuildSeries(IEnumerable runs, Color color) {
var values = new Dictionary>();
foreach (var run in runs) {
foreach (var step in runData[run]) {
List vals;
if (!values.TryGetValue(step.Item1, out vals)) {
vals = new List();
values.Add(step.Item1, vals);
}
vals.Add(step.Item2);
}
}
List series = new List();
var minMaxSeries = new Series();
var minMaxSeriesInfo = new SeriesInfo(Color.FromArgb(25, color), SeriesTypes.MinMax);
minMaxSeries.ChartType = SeriesChartType.Range;
minMaxSeries.Color = minMaxCheckBox.Checked ? minMaxSeriesInfo.Color : Color.Transparent;
minMaxSeries.IsVisibleInLegend = false;
minMaxSeries.Tag = minMaxSeriesInfo;
series.Add(minMaxSeries);
var quartilesSeries = new Series();
var quartilesSeriesInfo = new SeriesInfo(Color.FromArgb(50, color), SeriesTypes.Quartiles);
quartilesSeries.ChartType = SeriesChartType.Range;
quartilesSeries.Color = quartilesCheckBox.Checked ? quartilesSeriesInfo.Color : Color.Transparent;
quartilesSeries.IsVisibleInLegend = false;
quartilesSeries.Tag = quartilesSeriesInfo;
series.Add(quartilesSeries);
var medianSeries = new Series();
var medianSeriesInfo = new SeriesInfo(color, SeriesTypes.Median);
medianSeries.ChartType = SeriesChartType.FastLine;
medianSeries.Color = medianCheckBox.Checked ? medianSeriesInfo.Color : Color.Transparent;
medianSeries.BorderWidth = 3;
medianSeries.BorderDashStyle = ChartDashStyle.Solid;
medianSeries.IsVisibleInLegend = false;
medianSeries.Tag = medianSeriesInfo;
series.Add(medianSeries);
var averageSeries = new Series();
var averageSeriesInfo = new SeriesInfo(color, SeriesTypes.Average);
averageSeries.ChartType = SeriesChartType.FastLine;
averageSeries.Color = averageCheckBox.Checked ? averageSeriesInfo.Color : Color.Transparent;
averageSeries.BorderWidth = 3;
averageSeries.BorderDashStyle = ChartDashStyle.Dash;
averageSeries.IsVisibleInLegend = false;
averageSeries.Tag = averageSeriesInfo;
series.Add(averageSeries);
foreach (var point in values.OrderBy(x => x.Key)) {
if (point.Value.Count > 0) {
minMaxSeries.Points.Add(new DataPoint(point.Key, new double[] { point.Value.Min(), point.Value.Max() }));
quartilesSeries.Points.Add(new DataPoint(point.Key, new double[] { point.Value.Quantile(0.25), point.Value.Quantile(0.75) }));
medianSeries.Points.Add(new DataPoint(point.Key, point.Value.Median()));
averageSeries.Points.Add(new DataPoint(point.Key, point.Value.Average()));
}
}
return series;
}
#region Control Events
#region stepSizeTextBox
private void stepSizeTextBox_KeyDown(object sender, KeyEventArgs e) {
if ((e.KeyCode == Keys.Enter) || (e.KeyCode == Keys.Return))
stepSizeLabel.Select(); // select label to validate data
if (e.KeyCode == Keys.Escape) {
stepSizeTextBox.Text = stepSize.ToString();
stepSizeLabel.Select(); // select label to validate data
}
}
private void stepSizeTextBox_Validating(object sender, CancelEventArgs e) {
int val;
if (!int.TryParse(stepSizeTextBox.Text, out val) || val <= 0) {
e.Cancel = true;
errorProvider.SetError(stepSizeTextBox, "Invalid Value (Valid Value Format: \"[+]digits\")");
stepSizeTextBox.SelectAll();
}
}
private void stepSizeTextBox_Validated(object sender, EventArgs e) {
int val = int.Parse(stepSizeTextBox.Text);
errorProvider.SetError(stepSizeTextBox, string.Empty);
stepSizeTextBox.Text = val.ToString();
if (stepSize != val) {
stepSize = val;
CollectRunData();
UpdateChart();
}
}
#endregion
#region logScalingCheckBox
private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
chart.ChartAreas[0].AxisX.IsLogarithmic = logScalingCheckBox.Checked;
}
#endregion
#region chart
private void chart_MouseDown(object sender, MouseEventArgs e) {
HitTestResult result = chart.HitTest(e.X, e.Y);
if (result.ChartElementType == ChartElementType.LegendItem) {
var legendItemInfo = (result.Object as LegendItem).Tag as LegendItemInfo;
legendItemInfo.SeriesVisible = !legendItemInfo.SeriesVisible;
UpdateSeriesVisibility();
}
}
private void chart_MouseMove(object sender, MouseEventArgs e) {
HitTestResult result = chart.HitTest(e.X, e.Y);
switch (result.ChartElementType) {
case ChartElementType.LegendItem:
Cursor = Cursors.Hand;
break;
default:
Cursor = Cursors.Default;
break;
}
}
private void chart_CustomizeLegend(object sender, CustomizeLegendEventArgs e) {
foreach (LegendItem legendItem in e.LegendItems) {
var legendItemInfo = legendItem.Tag as LegendItemInfo;
legendItem.Color = legendItemInfo.SeriesVisible ? legendItemInfo.Color : Color.Transparent;
foreach (LegendCell cell in legendItem.Cells) {
cell.ForeColor = legendItemInfo.SeriesVisible ? Color.Black : Color.Gray;
}
}
}
#endregion
#region parametersTreeView
private void parametersTreeView_AfterSelect(object sender, TreeViewEventArgs e) {
addGroupButton.Enabled = (parametersTreeView.SelectedNode != null) && (parametersTreeView.SelectedNode.Parent != null);
}
private void parametersTreeView_AfterCheck(object sender, TreeViewEventArgs e) {
UpdateChart();
}
#endregion
#region groupsTreeView
private void groupsTreeView_AfterSelect(object sender, TreeViewEventArgs e) {
removeGroupButton.Enabled = (groupsTreeView.SelectedNode != null) && (groupsTreeView.SelectedNode.Parent != null);
}
private void groupsTreeView_AfterCheck(object sender, TreeViewEventArgs e) {
UpdateChart();
}
#endregion
#region addGroupButton, removeGroupButton
private void addGroupButton_Click(object sender, EventArgs e) {
var group = groupsTreeView.SelectedNode;
var param = parametersTreeView.SelectedNode;
var groupRuns = group.Parent == null ? group.Tag as IEnumerable : (group.Tag as GroupInfo).Runs;
if (param.Tag is ParameterInfo) {
var paramInfo = param.Tag as ParameterInfo;
var paramNode = new TreeNode();
foreach (var valueInfo in paramInfo.Values.Values) {
var valueRuns = groupRuns.Intersect(valueInfo.Runs);
var valueNode = new TreeNode(valueInfo.Value + " (" + valueRuns.Count() + ")");
valueNode.Tag = new GroupInfo(valueInfo.Value, valueInfo.Color, valueRuns, false);
paramNode.Nodes.Add(valueNode);
}
var paramRuns = groupRuns.Intersect(paramInfo.Runs);
paramNode.Text = paramInfo.Name + " (" + paramRuns.Count() + ")";
paramNode.Tag = new GroupInfo(paramInfo.Name, Color.Empty, paramRuns, true);
group.Nodes.Add(paramNode);
} else if (param.Tag is ParameterValueInfo) {
var paramInfo = param.Parent.Tag as ParameterInfo;
var valueInfo = param.Tag as ParameterValueInfo;
var valueRuns = groupRuns.Intersect(valueInfo.Runs);
var paramRuns = groupRuns.Intersect(paramInfo.Runs);
var paramNode = new TreeNode(paramInfo.Name + " (" + paramRuns.Count() + ")");
var valueNode = new TreeNode(valueInfo.Value + " (" + valueRuns.Count() + ")");
valueNode.Tag = new GroupInfo(valueInfo.Value, valueInfo.Color, valueRuns, false);
paramNode.Tag = new GroupInfo(paramInfo.Name, Color.Empty, paramRuns, true);
paramNode.Nodes.Add(valueNode);
group.Nodes.Add(paramNode);
}
groupsTreeView.Nodes[0].Expand();
}
private void removeGroupButton_Click(object sender, EventArgs e) {
if (groupsTreeView.SelectedNode != null)
groupsTreeView.SelectedNode.Remove();
UpdateChart();
}
#endregion
#region minMaxCheckBox, quartilesCheckBox, averageCheckBox, medianCheckBox
private void dataRowCheckBox_CheckedChanged(object sender, EventArgs e) {
UpdateSeriesVisibility();
}
#endregion
#endregion
#region Helpers
private IEnumerable TraverseTreeNodes(TreeNodeCollection nodes) {
foreach (var node in nodes.OfType()) {
yield return node;
foreach (var child in TraverseTreeNodes(node.Nodes))
yield return child;
}
}
#endregion
#region Inner Types
class ParameterInfo {
public string Name { get; private set; }
public Dictionary Values { get; private set; }
public IEnumerable Runs {
get { return Values.Values.SelectMany(x => x.Runs); }
}
public int RunCount {
get { return Values.Values.Select(x => x.RunCount).Sum(); }
}
public ParameterInfo(string name, string value, IRun run) {
Name = name;
Values = new Dictionary();
AddValue(value, run);
}
public void AddValue(string value, IRun run) {
ParameterValueInfo valueInfo;
if (!Values.TryGetValue(value, out valueInfo)) {
Values.Add(value, new ParameterValueInfo(value, run));
} else {
valueInfo.Runs.Add(run);
}
}
}
class ParameterValueInfo {
public string Value { get; private set; }
public Color Color { get; set; }
public IList Runs { get; private set; }
public int RunCount {
get { return Runs.Count; }
}
public ParameterValueInfo(string value, IRun run) {
Value = value;
Runs = new List();
Runs.Add(run);
}
public ParameterValueInfo(string value, Color color, IRun run) : this(value, run) {
Color = color;
}
}
class GroupInfo {
public string Text { get; private set; }
public Color Color { get; private set; }
public IList Runs { get; private set; }
public bool IsParameter { get; private set; }
public GroupInfo(string text, Color color, IEnumerable runs, bool isParamter) {
Text = text;
Color = color;
Runs = runs.ToList();
IsParameter = isParamter;
}
}
enum SeriesTypes {
MinMax,
Quartiles,
Average,
Median
}
class SeriesInfo {
public Color Color { get; private set; }
public SeriesTypes Type { get; private set; }
public SeriesInfo(Color color, SeriesTypes type) {
Color = color;
Type = type;
}
}
class LegendItemInfo {
public Color Color { get; private set; }
public bool SeriesVisible { get; set; }
public IEnumerable Series { get; private set; }
public LegendItemInfo(Color color, IEnumerable series) {
Color = color;
SeriesVisible = true;
Series = series;
}
}
#endregion
}
}