source: branches/PerformanceComparison/HeuristicLab.Optimization.Views/3.3/RunCollectionViews/RunCollectionRLDView.cs @ 12841

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Drawing;
using System.Globalization;
using System.Linq;
using System.Windows.Forms;
using HeuristicLab.Analysis;
using HeuristicLab.Collections;
using HeuristicLab.Core.Views;
using HeuristicLab.Data;
using HeuristicLab.MainForm;
using HeuristicLab.MainForm.WindowsForms;

namespace HeuristicLab.Optimization.Views {
  [View("Run-length Distribution View")]
  [Content(typeof(RunCollection), false)]
  public partial class RunCollectionRLDView : ItemView {
    private const string AllRuns = "All Runs";

    private static readonly Color[] colors = new[] {
      Color.FromArgb(0x40, 0x6A, 0xB7),
      Color.FromArgb(0xB1, 0x6D, 0x01),
      Color.FromArgb(0x4E, 0x8A, 0x06),
      Color.FromArgb(0x75, 0x50, 0x7B),
      Color.FromArgb(0x72, 0x9F, 0xCF),
      Color.FromArgb(0xA4, 0x00, 0x00),
      Color.FromArgb(0xAD, 0x7F, 0xA8),
      Color.FromArgb(0x29, 0x50, 0xCF),
      Color.FromArgb(0x90, 0xB0, 0x60),
      Color.FromArgb(0xF5, 0x89, 0x30),
      Color.FromArgb(0x55, 0x57, 0x53),
      Color.FromArgb(0xEF, 0x59, 0x59),
      Color.FromArgb(0xED, 0xD4, 0x30),
      Color.FromArgb(0x63, 0xC2, 0x16),
    };
    private static readonly DataRowVisualProperties.DataRowLineStyle[] lineStyles = new[] {
      DataRowVisualProperties.DataRowLineStyle.Solid,
      DataRowVisualProperties.DataRowLineStyle.Dash,
      DataRowVisualProperties.DataRowLineStyle.DashDot,
      DataRowVisualProperties.DataRowLineStyle.Dot
    };

    public new RunCollection Content {
      get { return (RunCollection)base.Content; }
      set { base.Content = value; }
    }

    private double[] targets;
    private double[] budgets;

    private bool suppressUpdates;
    private readonly IndexedDataTable<double> byTargetDataTable;
    public IndexedDataTable<double> ByTargetDataTable {
      get { return byTargetDataTable; }
    }
    private readonly IndexedDataTable<double> byCostDataTable;
    public IndexedDataTable<double> ByCostDataTable {
      get { return byCostDataTable; }
    }

    public RunCollectionRLDView() {
      InitializeComponent();
      byTargetDataTable = new IndexedDataTable<double>("ECDF by Target", "A data table containing the ECDF of each of a number of groups.") {
        VisualProperties = {
          YAxisTitle = "Proportion of reached targets",
          YAxisMinimumFixedValue = 0, YAxisMinimumAuto = false,
          YAxisMaximumFixedValue = 1, YAxisMaximumAuto = false
        }
      };
      byTargetViewHost.Content = byTargetDataTable;
      byCostDataTable = new IndexedDataTable<double>("ECDF by Cost", "A data table containing the ECDF of each of a number of groups.") {
        VisualProperties = {
          YAxisTitle = "Proportion of required budgets",
          YAxisMinimumFixedValue = 0, YAxisMinimumAuto = false,
          YAxisMaximumFixedValue = 1, YAxisMaximumAuto = false
        }
      };
      byCostViewHost.Content = byCostDataTable;
      suppressUpdates = false;
    }

    #region Content events
    protected override void RegisterContentEvents() {
      base.RegisterContentEvents();
      Content.ItemsAdded += Content_ItemsAdded;
      Content.ItemsRemoved += Content_ItemsRemoved;
      Content.CollectionReset += Content_CollectionReset;
      Content.UpdateOfRunsInProgressChanged += Content_UpdateOfRunsInProgressChanged;
      Content.OptimizerNameChanged += Content_AlgorithmNameChanged;
    }
    protected override void DeregisterContentEvents() {
      Content.ItemsAdded -= Content_ItemsAdded;
      Content.ItemsRemoved -= Content_ItemsRemoved;
      Content.CollectionReset -= Content_CollectionReset;
      Content.UpdateOfRunsInProgressChanged -= Content_UpdateOfRunsInProgressChanged;
      Content.OptimizerNameChanged -= Content_AlgorithmNameChanged;
      base.DeregisterContentEvents();
    }

    private void Content_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      if (suppressUpdates) return;
      if (InvokeRequired) {
        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsAdded), sender, e);
        return;
      }
      UpdateGroupAndProblemComboBox();
      UpdateDataTableComboBox();
      foreach (var run in e.Items)
        RegisterRunEvents(run);
    }
    private void Content_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      if (suppressUpdates) return;
      if (InvokeRequired) {
        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_ItemsRemoved), sender, e);
        return;
      }
      UpdateGroupAndProblemComboBox();
      UpdateDataTableComboBox();
      foreach (var run in e.Items)
        DeregisterRunEvents(run);
    }
    private void Content_CollectionReset(object sender, CollectionItemsChangedEventArgs<IRun> e) {
      if (suppressUpdates) return;
      if (InvokeRequired) {
        Invoke(new CollectionItemsChangedEventHandler<IRun>(Content_CollectionReset), sender, e);
        return;
      }
      UpdateGroupAndProblemComboBox();
      UpdateDataTableComboBox();
      foreach (var run in e.OldItems)
        DeregisterRunEvents(run);
    }
    private void Content_AlgorithmNameChanged(object sender, EventArgs e) {
      if (InvokeRequired)
        Invoke(new EventHandler(Content_AlgorithmNameChanged), sender, e);
      else UpdateCaption();
    }
    private void Content_UpdateOfRunsInProgressChanged(object sender, EventArgs e) {
      if (InvokeRequired) {
        Invoke(new EventHandler(Content_UpdateOfRunsInProgressChanged), sender, e);
        return;
      }
      suppressUpdates = Content.UpdateOfRunsInProgress;
      if (!suppressUpdates) {
        UpdateDataTableComboBox();
        UpdateGroupAndProblemComboBox();
        UpdateRuns();
      }
    }

    private void RegisterRunEvents(IRun run) {
      run.PropertyChanged += run_PropertyChanged;
    }
    private void DeregisterRunEvents(IRun run) {
      run.PropertyChanged -= run_PropertyChanged;
    }
    private void run_PropertyChanged(object sender, PropertyChangedEventArgs e) {
      if (suppressUpdates) return;
      if (InvokeRequired) {
        Invoke((Action<object, PropertyChangedEventArgs>)run_PropertyChanged, sender, e);
      } else {
        if (e.PropertyName == "Visible")
          UpdateRuns();
      }
    }
    #endregion

    protected override void OnContentChanged() {
      base.OnContentChanged();
      dataTableComboBox.Items.Clear();
      groupComboBox.Items.Clear();
      byTargetDataTable.Rows.Clear();

      UpdateCaption();
      if (Content != null) {
        UpdateGroupAndProblemComboBox();
        UpdateDataTableComboBox();
      }
    }


    private void UpdateGroupAndProblemComboBox() {
      var selectedGroupItem = (string)groupComboBox.SelectedItem;

      var groupings = Content.ParameterNames.OrderBy(x => x).ToArray();
      groupComboBox.Items.Clear();
      groupComboBox.Items.Add(AllRuns);
      groupComboBox.Items.AddRange(groupings);
      if (selectedGroupItem != null && groupComboBox.Items.Contains(selectedGroupItem)) {
        groupComboBox.SelectedItem = selectedGroupItem;
      } else if (groupComboBox.Items.Count > 0) {
        groupComboBox.SelectedItem = groupComboBox.Items[0];
      }

      var problems = new HashSet<ProblemDescription>();
      foreach (var run in Content) {
        problems.Add(new ProblemDescription(run));
      }

      var problemTypesDifferent = problems.Select(x => x.ProblemType).Distinct().Count() > 1;
      var problemNamesDifferent = problems.Select(x => x.ProblemName).Distinct().Count() > 1;
      var evaluatorDifferent = problems.Select(x => x.Evaluator).Distinct().Count() > 1;
      var allEqual = !problemTypesDifferent && !problemNamesDifferent && !evaluatorDifferent;

      var selectedProblemItem = (ProblemDescription)problemComboBox.SelectedItem;
      problemComboBox.Items.Clear();
      foreach (var prob in problems.OrderBy(x => x.ToString()).ToList()) {
        prob.DisplayProblemType = problemTypesDifferent;
        prob.DisplayProblemName = problemNamesDifferent || allEqual;
        prob.DisplayEvaluator = evaluatorDifferent;
        problemComboBox.Items.Add(prob);
        if (prob.Equals(selectedProblemItem)) problemComboBox.SelectedItem = prob;
      }
      SetEnabledStateOfControls();
    }

    private void UpdateDataTableComboBox() {
      string selectedItem = (string)dataTableComboBox.SelectedItem;

      dataTableComboBox.Items.Clear();
      var dataTables = (from run in Content
                        from result in run.Results
                        where result.Value is IndexedDataTable<double>
                        select result.Key).Distinct().ToArray();

      dataTableComboBox.Items.AddRange(dataTables);
      if (selectedItem != null && dataTableComboBox.Items.Contains(selectedItem)) {
        dataTableComboBox.SelectedItem = selectedItem;
      } else if (dataTableComboBox.Items.Count > 0) {
        dataTableComboBox.SelectedItem = dataTableComboBox.Items[0];
      }
    }

    protected override void SetEnabledStateOfControls() {
      base.SetEnabledStateOfControls();
      groupComboBox.Enabled = Content != null;
      problemComboBox.Enabled = Content != null && problemComboBox.Items.Count > 1;
      dataTableComboBox.Enabled = Content != null && dataTableComboBox.Items.Count > 1;
      addTargetsAsResultButton.Enabled = Content != null && targets != null && dataTableComboBox.SelectedIndex >= 0;
      addBudgetsAsResultButton.Enabled = Content != null && budgets != null && dataTableComboBox.SelectedIndex >= 0;
    }

    private List<Tuple<string, List<IRun>>> GroupRuns() {
      var groupedRuns = new List<Tuple<string, List<IRun>>>();

      var table = (string)dataTableComboBox.SelectedItem;
      if (string.IsNullOrEmpty(table)) return groupedRuns;

      var selectedGroup = (string)groupComboBox.SelectedItem;
      if (string.IsNullOrEmpty(selectedGroup)) return groupedRuns;

      var selectedProblem = (ProblemDescription)problemComboBox.SelectedItem;
      if (selectedProblem == null) selectedProblem = ProblemDescription.MatchAll;

      if (selectedGroup == AllRuns)
        groupedRuns = new List<Tuple<string, List<IRun>>> {
          Tuple.Create(AllRuns, Content.Where(r => selectedProblem.Match(r) && r.Results.ContainsKey(table) && r.Visible).ToList())
        };
      else
        groupedRuns = (from r in Content
                       where r.Parameters.ContainsKey(selectedGroup)
                             && selectedProblem.Match(r)
                             && r.Results.ContainsKey(table)
                             && r.Visible
                       group r by r.Parameters[selectedGroup].ToString()
                         into g
                         select Tuple.Create(g.Key, g.ToList())).ToList();

      return groupedRuns;
    }

    private void UpdateRuns() {
      if (InvokeRequired) {
        Invoke((Action)UpdateRuns);
        return;
      }
      SuspendRepaint();
      try {
        UpdateResultsByTarget();
        UpdateResultsByCost();
      } finally { ResumeRepaint(true); }
    }

    #region Performance analysis by (multiple) target(s)
    private void UpdateResultsByTarget() {
      // necessary to reset log scale -> empty chart cannot use log scaling
      byTargetDataTable.VisualProperties.XAxisLogScale = false;
      byTargetDataTable.Rows.Clear();

      var table = (string)dataTableComboBox.SelectedItem;
      if (string.IsNullOrEmpty(table)) return;

      if (targets == null) GenerateDefaultTargets(table);

      var groupedRuns = GroupRuns();
      if (groupedRuns.Count == 0) return;

      var xAxisTitles = new HashSet<string>();
      var colorCount = 0;
      var lineStyleCount = 0;
      var maximization = IsMaximization();

      foreach (var group in groupedRuns) {
        var hits = new Dictionary<string, SortedList<double, double>>();
        var maxLength = 0.0;

        foreach (var run in group.Item2) {
          var resultsTable = (IndexedDataTable<double>)run.Results[table];
          xAxisTitles.Add(resultsTable.VisualProperties.XAxisTitle);

          if (eachOrAllTargetCheckBox.Checked) {
            CalculateHitsForEachTarget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
          } else {
            maxLength = CalculateHitsForAllTargets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
          }
        }

        foreach (var list in hits) {
          var row = new IndexedDataRow<double>(list.Key) {
            VisualProperties = {
              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
              LineWidth = 2,
              Color = colors[colorCount],
              LineStyle = lineStyles[lineStyleCount]
            }
          };

          var total = 0.0;
          foreach (var h in list.Value) {
            total += h.Value;
            row.Values.Add(Tuple.Create(h.Key, total));
          }

          if (maxLength > 0 && (row.Values.Count == 0 || row.Values.Last().Item1 < maxLength))
            row.Values.Add(Tuple.Create(maxLength, total));

          byTargetDataTable.Rows.Add(row);
        }
        colorCount = (colorCount + 1) % colors.Length;
        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
      }

      byTargetDataTable.VisualProperties.XAxisTitle = string.Join(" / ", xAxisTitles);
      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;

      UpdateErtTables(groupedRuns);
    }

    private void GenerateDefaultTargets(string table) {
      var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
      var worst = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Min()).Min();
      var best = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item2).Max()).Max();
      var maximization = IsMaximization();
      if (!maximization) {
        var h = worst;
        worst = best;
        best = h;
      }

      if (best == 0 || Math.Abs(best - worst) < Math.Abs(best * 2))
        targets = Enumerable.Range(0, 11).Select(x => worst + (x / 10.0) * (best - worst)).ToArray();
      else if (best > 0) targets = Enumerable.Range(0, 11).Select(x => best * (1.0 + (10 - x) / 10.0)).ToArray();
      else if (best < 0) targets = Enumerable.Range(0, 11).Select(x => best / (1.0 + (10 - x) / 10.0)).ToArray();

      suppressTargetsEvents = true;
      targetsTextBox.Text = string.Join(" ; ", targets);
      suppressTargetsEvents = false;
    }

    private void CalculateHitsForEachTarget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
      foreach (var l in targets) {
        var key = groupName + "-" + l;
        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
        foreach (var v in row.Values) {
          if (maximization && v.Item2 >= l || !maximization && v.Item2 <= l) {
            if (hits[key].ContainsKey(v.Item1))
              hits[key][v.Item1] += 1.0 / groupCount;
            else hits[key][v.Item1] = 1.0 / groupCount;
            break;
          }
        }
      }
    }

    private double CalculateHitsForAllTargets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
      var values = row.Values;
      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());

      var i = 0;
      var j = 0;
      while (i < targets.Length && j < values.Count) {
        var current = values[j];
        if (maximization && current.Item2 >= targets[i]
          || !maximization && current.Item2 <= targets[i]) {
          if (!hits[groupName].ContainsKey(current.Item1)) hits[groupName][current.Item1] = 0;
          hits[groupName][current.Item1] += 1.0 / (groupCount * targets.Length);
          i++;
        } else {
          j++;
        }
      }
      if (j == values.Count) j--;
      return values[j].Item1;
    }

    private void UpdateErtTables(List<Tuple<string, List<IRun>>> groupedRuns) {
      ertViewHost.Content = null;
      var columns = 1 + targets.Length + 1;
      var matrix = new string[groupedRuns.Count + 1, columns];
      var rowCount = 0;
      var maximization = IsMaximization();

      var colNames = new string[columns];
      colNames[0] = colNames[colNames.Length - 1] = string.Empty;
      for (var i = 0; i < targets.Length; i++) {
        colNames[i + 1] = maximization ? (targets[targets.Length - 1] / targets[i] - 1).ToString("0.0%")
                                       : (targets[i] / targets[targets.Length - 1] - 1).ToString("0.0%");
        matrix[rowCount, i + 1] = targets[i].ToString(CultureInfo.CurrentCulture.NumberFormat);
      }
      matrix[rowCount, columns - 1] = "#succ";
      rowCount++;

      foreach (var group in groupedRuns) {
        matrix[rowCount, 0] = group.Item1;
        var bestSucc = string.Empty;
        for (var i = 0; i < targets.Length; i++) {
          string succ;
          matrix[rowCount, i + 1] = CalculateExpectedRunTime(group.Item2, targets[i], maximization, out succ);
          if (i == targets.Length - 1) bestSucc = succ;
        }
        matrix[rowCount, columns - 1] = bestSucc;
        rowCount++;
      }
      ertViewHost.Content = new StringMatrix(matrix) { ColumnNames = colNames };
    }

    private string CalculateExpectedRunTime(List<IRun> group, double target, bool maximization, out string successProb) {
      var table = (string)dataTableComboBox.SelectedItem;
      successProb = "-";
      if (string.IsNullOrEmpty(table)) return "N/A";
      var successful = new List<double>();
      var unsuccessful = new List<double>();
      foreach (var r in group) {
        var targetAchieved = false;
        var row = ((IndexedDataTable<double>)r.Results[table]).Rows.First();
        foreach (var v in row.Values) {
          if (maximization && v.Item2 >= target || !maximization && v.Item2 <= target) {
            successful.Add(v.Item1);
            targetAchieved = true;
            break;
          }
        }
        if (!targetAchieved) unsuccessful.Add(row.Values.Last().Item1);
      }
      successProb = successful.Count + "/" + (successful.Count + unsuccessful.Count);
      if (successful.Count == 0) return "\u221e"; // infinity symbol
      if (unsuccessful.Count == 0) return successful.Average().ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);

      var ps = successful.Count / (double)(successful.Count + unsuccessful.Count);
      return (successful.Average() + ((1.0 - ps) / ps) * unsuccessful.Average()).ToString("##,0.0", CultureInfo.CurrentCulture.NumberFormat);
    }
    #endregion

    #region Performance analysis by (multiple) budget(s)
    private void UpdateResultsByCost() {
      // necessary to reset log scale -> empty chart cannot use log scaling
      byCostDataTable.VisualProperties.XAxisLogScale = false;
      byCostDataTable.Rows.Clear();

      var table = (string)dataTableComboBox.SelectedItem;
      if (string.IsNullOrEmpty(table)) return;

      if (budgets == null) GenerateDefaultBudgets(table);

      var groupedRuns = GroupRuns();
      if (groupedRuns.Count == 0) return;

      var colorCount = 0;
      var lineStyleCount = 0;
      var maximization = IsMaximization();

      foreach (var group in groupedRuns) {
        var hits = new Dictionary<string, SortedList<double, double>>();

        foreach (var run in group.Item2) {
          var resultsTable = (IndexedDataTable<double>)run.Results[table];

          if (eachOrAllBudgetsCheckBox.Checked) {
            CalculateHitsForEachBudget(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
          } else {
            CalculateHitsForAllBudgets(hits, resultsTable.Rows.First(), maximization, group.Item2.Count, group.Item1);
          }
        }

        foreach (var list in hits) {
          var row = new IndexedDataRow<double>(list.Key) {
            VisualProperties = {
              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
              LineWidth = 2,
              Color = colors[colorCount],
              LineStyle = lineStyles[lineStyleCount]
            }
          };

          var total = 0.0;
          foreach (var h in list.Value) {
            total += h.Value;
            row.Values.Add(Tuple.Create(h.Key, total));
          }

          byCostDataTable.Rows.Add(row);
        }
        colorCount = (colorCount + 1) % colors.Length;
        if (colorCount == 0) lineStyleCount = (lineStyleCount + 1) % lineStyles.Length;
      }

      byCostDataTable.VisualProperties.XAxisTitle = "Targets";
      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
    }

    private void GenerateDefaultBudgets(string table) {
      var runs = GroupRuns().SelectMany(x => x.Item2).ToList();
      var min = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Min()).Min();
      var max = runs.Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Select(y => y.Item1).Max()).Max();

      var maxMagnitude = (int)Math.Ceiling(Math.Log10(max));
      var minMagnitude = (int)Math.Floor(Math.Log10(min));
      if (maxMagnitude - minMagnitude >= 3) {
        budgets = new double[maxMagnitude - minMagnitude];
        for (var i = minMagnitude; i < maxMagnitude; i++) {
          budgets[i - minMagnitude] = Math.Pow(10, i);
        }
      } else {
        var range = max - min;
        budgets = Enumerable.Range(0, 6).Select(x => min + (x / 5.0) * range).ToArray();
      }
      suppressBudgetsEvents = true;
      budgetsTextBox.Text = string.Join(" ; ", budgets);
      suppressBudgetsEvents = false;
    }

    private void CalculateHitsForEachBudget(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
      foreach (var b in budgets) {
        var key = groupName + "-" + b;
        if (!hits.ContainsKey(key)) hits.Add(key, new SortedList<double, double>());
        Tuple<double, double> prev = null;
        foreach (var v in row.Values) {
          if (v.Item1 >= b) {
            // the budget may be too low to achieve any target
            if (prev == null && v.Item1 != b) break;
            var tgt = (prev == null || v.Item1 == b) ? v.Item2 : prev.Item2;
            if (hits[key].ContainsKey(tgt))
              hits[key][tgt] += 1.0 / groupCount;
            else hits[key][tgt] = 1.0 / groupCount;
            break;
          }
          prev = v;
        }
        if (hits[key].Count == 0) hits.Remove(key);
      }
    }

    private void CalculateHitsForAllBudgets(Dictionary<string, SortedList<double, double>> hits, IndexedDataRow<double> row, bool maximization, int groupCount, string groupName) {
      var values = row.Values;
      if (!hits.ContainsKey(groupName)) hits.Add(groupName, new SortedList<double, double>());

      var i = 0;
      var j = 0;
      Tuple<double, double> prev = null;
      while (i < budgets.Length && j < values.Count) {
        var current = values[j];
        if (current.Item1 >= budgets[i]) {
          if (prev != null || current.Item1 == budgets[i]) {
            var tgt = (prev == null || current.Item1 == budgets[i]) ? current.Item2 : prev.Item2;
            if (!hits[groupName].ContainsKey(tgt)) hits[groupName][tgt] = 0;
            hits[groupName][tgt] += 1.0 / (groupCount * budgets.Length);
          }
          i++;
        } else {
          j++;
          prev = current;
        }
      }
      var lastTgt = values.Last().Item2;
      if (i < budgets.Length && !hits[groupName].ContainsKey(lastTgt)) hits[groupName][lastTgt] = 0;
      while (i < budgets.Length) {
        hits[groupName][lastTgt] += 1.0 / (groupCount * budgets.Length);
        i++;
      }
    }
    #endregion

    private void UpdateCaption() {
      Caption = Content != null ? Content.OptimizerName + " RLD View" : ViewAttribute.GetViewName(GetType());
    }

    private void groupComboBox_SelectedIndexChanged(object sender, EventArgs e) {
      UpdateRuns();
      SetEnabledStateOfControls();
    }
    private void problemComboBox_SelectedIndexChanged(object sender, EventArgs e) {
      if (dataTableComboBox.SelectedIndex >= 0)
        GenerateDefaultTargets((string)dataTableComboBox.SelectedItem);
      UpdateRuns();
      SetEnabledStateOfControls();
    }
    private void dataTableComboBox_SelectedIndexChanged(object sender, EventArgs e) {
      if (dataTableComboBox.SelectedIndex >= 0)
        GenerateDefaultBudgets((string)dataTableComboBox.SelectedItem);
      UpdateRuns();
      SetEnabledStateOfControls();
    }

    private void logScalingCheckBox_CheckedChanged(object sender, EventArgs e) {
      byTargetDataTable.VisualProperties.XAxisLogScale = byTargetDataTable.Rows.Count > 0 && targetLogScalingCheckBox.Checked;
      byCostDataTable.VisualProperties.XAxisLogScale = byCostDataTable.Rows.Count > 0 && budgetLogScalingCheckBox.Checked;
    }

    #region Event handlers for target analysis
    private bool suppressTargetsEvents;
    private void targetsTextBox_Validating(object sender, CancelEventArgs e) {
      if (suppressTargetsEvents) return;
      var targetStrings = targetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
      var targetList = new List<double>();
      foreach (var ts in targetStrings) {
        double t;
        if (!double.TryParse(ts, out t)) {
          errorProvider.SetError(targetsTextBox, "Not all targets can be parsed: " + ts);
          e.Cancel = true;
          return;
        }
        targetList.Add(t);
      }
      if (targetList.Count == 0) {
        errorProvider.SetError(targetsTextBox, "Give at least one target value!");
        e.Cancel = true;
        return;
      }
      e.Cancel = false;
      errorProvider.SetError(targetsTextBox, null);
      targets = targetList.ToArray();
      UpdateResultsByTarget();
      SetEnabledStateOfControls();
    }

    private void eachOrAllTargetCheckBox_CheckedChanged(object sender, EventArgs e) {
      var each = eachOrAllTargetCheckBox.Checked;
      eachOrAllTargetCheckBox.Text = each ? "each" : "all";
      SuspendRepaint();
      try {
        UpdateResultsByTarget();
      } finally { ResumeRepaint(true); }
    }

    private void generateTargetsButton_Click(object sender, EventArgs e) {
      var maximization = IsMaximization();
      decimal max = 1, min = 0, count = 10;
      if (targets != null) {
        max = (decimal)targets.Max();
        min = (decimal)targets.Min();
        count = targets.Length;
      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
        var table = (string)dataTableComboBox.SelectedItem;
        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item2)).Max();
        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item2)).Min();
        count = 6;
      }
      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
        if (dialog.ShowDialog() == DialogResult.OK) {
          if (dialog.Values.Any()) {
            targets = maximization ? dialog.Values.Select(x => (double)x).ToArray()
                                  : dialog.Values.Reverse().Select(x => (double)x).ToArray();
            suppressTargetsEvents = true;
            targetsTextBox.Text = string.Join(" ; ", targets);
            suppressTargetsEvents = false;

            UpdateResultsByTarget();
            SetEnabledStateOfControls();
          }
        }
      }
    }

    private void addTargetsAsResultButton_Click(object sender, EventArgs e) {
      var table = (string)dataTableComboBox.SelectedItem;
      var maximization = IsMaximization();
      foreach (var run in Content) {
        if (!run.Results.ContainsKey(table)) continue;
        var resultsTable = (IndexedDataTable<double>)run.Results[table];
        var values = resultsTable.Rows.First().Values;
        var i = 0;
        var j = 0;
        while (i < targets.Length && j < values.Count) {
          var current = values[j];
          if (maximization && current.Item2 >= targets[i]
              || !maximization && current.Item2 <= targets[i]) {
            run.Results[table + ".Target" + targets[i]] = new DoubleValue(current.Item1);
            i++;
          } else {
            j++;
          }
        }
      }
    }
    #endregion

    #region Event handlers for cost analysis
    private bool suppressBudgetsEvents;
    private void budgetsTextBox_Validating(object sender, CancelEventArgs e) {
      if (suppressBudgetsEvents) return;
      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
      var budgetList = new List<double>();
      foreach (var ts in budgetStrings) {
        double b;
        if (!double.TryParse(ts, out b)) {
          errorProvider.SetError(budgetsTextBox, "Not all targets can be parsed: " + ts);
          e.Cancel = true;
          return;
        }
        budgetList.Add(b);
      }
      if (budgetList.Count == 0) {
        errorProvider.SetError(budgetsTextBox, "Give at least one target value!");
        e.Cancel = true;
        return;
      }
      e.Cancel = false;
      errorProvider.SetError(budgetsTextBox, null);
      budgets = budgetList.ToArray();
      UpdateResultsByCost();
      SetEnabledStateOfControls();
    }

    private void eachOrAllBudgetsCheckBox_CheckedChanged(object sender, EventArgs e) {
      var each = eachOrAllBudgetsCheckBox.Checked;
      eachOrAllBudgetsCheckBox.Text = each ? "each" : "all";
      SuspendRepaint();
      try {
        UpdateResultsByCost();
      } finally { ResumeRepaint(true); }
    }

    private void generateBudgetsButton_Click(object sender, EventArgs e) {
      decimal max = 1, min = 0, count = 10;
      if (budgets != null) {
        max = (decimal)budgets.Max();
        min = (decimal)budgets.Min();
        count = budgets.Length;
      } else if (Content.Count > 0 && dataTableComboBox.SelectedIndex >= 0) {
        var table = (string)dataTableComboBox.SelectedItem;
        min = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Min(y => y.Item1)).Min();
        max = (decimal)Content.Where(x => x.Results.ContainsKey(table)).Select(x => ((IndexedDataTable<double>)x.Results[table]).Rows.First().Values.Max(y => y.Item1)).Max();
        count = 6;
      }
      using (var dialog = new DefineArithmeticProgressionDialog(false, min, max, (max - min) / count)) {
        if (dialog.ShowDialog() == DialogResult.OK) {
          if (dialog.Values.Any()) {
            budgets = dialog.Values.OrderBy(x => x).Select(x => (double)x).ToArray();

            suppressBudgetsEvents = true;
            budgetsTextBox.Text = string.Join(" ; ", budgets);
            suppressBudgetsEvents = false;

            UpdateResultsByCost();
            SetEnabledStateOfControls();
          }
        }
      }
    }

    private void addBudgetsAsResultButton_Click(object sender, EventArgs e) {
      var table = (string)dataTableComboBox.SelectedItem;
      var budgetStrings = budgetsTextBox.Text.Split(new[] { ';', '\t', ' ' }, StringSplitOptions.RemoveEmptyEntries);
      if (budgetStrings.Length == 0) {
        MessageBox.Show("Define a number of budgets.");
        return;
      }
      var budgetList = new List<double>();
      foreach (var bs in budgetStrings) {
        double v;
        if (!double.TryParse(bs, out v)) {
          MessageBox.Show("Budgets must be a valid number: " + bs);
          return;
        }
        budgetList.Add(v);
      }
      budgetList.Sort();

      foreach (var run in Content) {
        if (!run.Results.ContainsKey(table)) continue;
        var resultsTable = (IndexedDataTable<double>)run.Results[table];
        var values = resultsTable.Rows.First().Values;
        var i = 0;
        var j = 0;
        Tuple<double, double> prev = null;
        while (i < budgetList.Count && j < values.Count) {
          var current = values[j];
          if (current.Item1 >= budgetList[i]) {
            if (prev != null || current.Item1 == budgetList[i]) {
              var tgt = (prev == null || current.Item1 == budgetList[i]) ? current.Item2 : prev.Item2;
              run.Results[table + ".Cost" + budgetList[i]] = new DoubleValue(tgt);
            }
            i++;
          } else {
            j++;
            prev = current;
          }
        }
      }
    }
    #endregion

    #region Helpers
    // Determines if the RunCollection contains maximization or minimization runs
    private bool IsMaximization() {
      if (Content == null) return false;
      if (Content.Count > 0) {
        foreach (var run in Content.Where(x => x.Parameters.ContainsKey("Maximization")
                                            && x.Parameters["Maximization"] is BoolValue)) {
          if (((BoolValue)run.Parameters["Maximization"]).Value) {
            return true;
          } else {
            return false;
          }
        }
        if (dataTableComboBox.SelectedIndex >= 0) {
          var selectedTable = (string)dataTableComboBox.SelectedItem;
          foreach (var run in Content.Where(x => x.Results.ContainsKey(selectedTable))) {
            var table = run.Results[selectedTable] as IndexedDataTable<double>;
            if (table == null) continue;
            var firstRowValues = table.Rows.First().Values;
            if (firstRowValues.Count < 2) continue;
            return firstRowValues[0].Item2 < firstRowValues[firstRowValues.Count - 1].Item2;
          }
        }
      }
      // assume minimization by default
      return false;
    }
    #endregion


    private class ProblemDescription {
      private readonly bool matchAll;
      public static readonly ProblemDescription MatchAll = new ProblemDescription();

      private ProblemDescription() {
        ProblemType = string.Empty;
        ProblemName = string.Empty;
        Evaluator = string.Empty;
        DisplayProblemType = false;
        DisplayProblemName = false;
        DisplayEvaluator = false;
        matchAll = true;
      }

      public ProblemDescription(IRun run) {
        ProblemType = GetStringValueOrEmpty(run, "Problem Type");
        ProblemName = GetStringValueOrEmpty(run, "Problem Name");
        Evaluator = GetStringValueOrEmpty(run, "Evaluator");
        DisplayProblemType = !string.IsNullOrEmpty(ProblemType);
        DisplayProblemName = !string.IsNullOrEmpty(ProblemName);
        DisplayEvaluator = !string.IsNullOrEmpty(Evaluator);
        matchAll = false;
      }

      public bool DisplayProblemType { get; set; }
      public string ProblemType { get; set; }
      public bool DisplayProblemName { get; set; }
      public string ProblemName { get; set; }
      public bool DisplayEvaluator { get; set; }
      public string Evaluator { get; set; }

      public bool Match(IRun run) {
        return matchAll ||
               GetStringValueOrEmpty(run, "Problem Type") == ProblemType
               && GetStringValueOrEmpty(run, "Problem Name") == ProblemName
               && GetStringValueOrEmpty(run, "Evaluator") == Evaluator;
      }

      private string GetStringValueOrEmpty(IRun run, string key) {
        return run.Parameters.ContainsKey(key) ? ((StringValue)run.Parameters[key]).Value : string.Empty;
      }

      public override bool Equals(object obj) {
        var other = obj as ProblemDescription;
        if (other == null) return false;
        return ProblemType == other.ProblemType
               && ProblemName == other.ProblemName
               && Evaluator == other.Evaluator;
      }

      public override int GetHashCode() {
        return ProblemType.GetHashCode() ^ ProblemName.GetHashCode() ^ Evaluator.GetHashCode();
      }

      public override string ToString() {
        return string.Join("  --  ", new[] {
          (DisplayProblemType ? ProblemType : string.Empty),
          (DisplayProblemName ? ProblemName : string.Empty),
          (DisplayEvaluator ? Evaluator : string.Empty) }.Where(x => !string.IsNullOrEmpty(x)));
      }
    }
  }
}