Changeset 14526

Timestamp:

12/22/16 16:52:32 (7 years ago)

Author:

bburlacu

Message:

#2679: Add analyzer for SingleObjectiveGoalSeekingProblem. Add result aggregation in the GoalSeekingOptimizer. Remove unused dataset parameter from the problems.

Location:

branches/HeuristicLab.GoalSeekingProblem

Files:

• HeuristicLab.GoalSeekingProblem.Views/3.4/GoalSeekingOptimizerView.cs (modified) (2 diffs)
• HeuristicLab.GoalSeekingProblem.Views/3.4/GoalSeekingProblem.Views.csproj (modified) (2 diffs)
• HeuristicLab.GoalSeekingProblem/3.4/Analyzers/BestSolutionAnalyzer.cs (modified) (1 diff)
• HeuristicLab.GoalSeekingProblem/3.4/GoalSeekingOptimizer.cs (modified) (9 diffs)
• HeuristicLab.GoalSeekingProblem/3.4/GoalSeekingProblem.csproj (modified) (1 diff)
• HeuristicLab.GoalSeekingProblem/3.4/MultiObjectiveGoalSeekingProblem.cs (modified) (2 diffs)
• HeuristicLab.GoalSeekingProblem/3.4/Properties (modified) (1 prop)
• HeuristicLab.GoalSeekingProblem/3.4/Properties/AssemblyInfo.cs (deleted)
• HeuristicLab.GoalSeekingProblem/3.4/SingleObjectiveGoalSeekingProblem.cs (modified) (4 diffs)

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem.Views/3.4/GoalSeekingOptimizerView.cs

@@ -54 +54 @@
       if (Content.ProblemData != null)
         dataViewHost.Content = Content.ProblemData;
+
+      if (Content.Results != null)
+        resultsViewHost.Content = Content.Results;
     }
 
@@ -61 +64 @@
       Content.ProblemChanged += (o, e) => { problemViewHost.Content = Content.Problem; };
       Content.ProblemDataChanged += (o, e) => { dataViewHost.Content = Content.ProblemData; };
+      Content.ResultsChanged += (o, e) => { resultsViewHost.Content = Content.Results; };
     }
 

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem.Views/3.4/GoalSeekingProblem.Views.csproj

@@ -22 +22 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <LangVersion>4</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
@@ -32 +33 @@
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec" />
     <Reference Include="HeuristicLab.Common-3.3">
       <HintPath>..\..\..\..\trunk\sources\bin\HeuristicLab.Common-3.3.dll</HintPath>

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/Analyzers/BestSolutionAnalyzer.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/GoalSeekingOptimizer.cs

@@ -24 +24 @@
 using System.Drawing;
 using System.Linq;
+using HeuristicLab.Analysis;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
@@ -69 +71 @@
 
         optimizer = value;
-        optimizer.Problem = problem;
+
+        if (problem != null)
+          optimizer.Problem = problem;
+
         RegisterOptimizerEvents();
         OnAlgorithmChanged();
@@ -92 +97 @@
     }
 
+    [Storable]
+    private ResultCollection results;
+    public ResultCollection Results {
+      get { return results; }
+      private set {
+        if (value != null)
+          results = value;
+        OnResultsChanged();
+      }
+    }
+
     private GoalSeekingOptimizerAction gsoAction;
 
@@ -112 +128 @@
       this.Problem = (IGoalSeekingProblem)original.Problem.Clone(cloner);
       this.ProblemData = (IRegressionProblemData)original.ProblemData.Clone(cloner);
+      this.Results = (ResultCollection)original.Results.Clone();
     }
 
@@ -139 +156 @@
     }
 
+    public EventHandler ResultsChanged;
+    private void OnResultsChanged() {
+      var changed = ResultsChanged;
+      if (changed != null)
+        changed(this, EventArgs.Empty);
+    }
+
     private int calculatedRows;
+    private int currentRow;
+
     public void Start() {
       if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
@@ -148 +174 @@
         Optimizer.Prepare(clearRuns: true);
         calculatedRows = 0;
-
-        var row = problemData.TrainingIndices.Skip(calculatedRows).First();
-        problem.Configure(problemData, row);
+        currentRow = problemData.TrainingIndices.Skip(calculatedRows).First();
+        problem.Configure(problemData, currentRow);
         optimizer.Prepare();
       }
@@ -188 +213 @@
       if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped))
         throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
+
+      if (Results != null)
+        Results.Clear();
+
       if (optimizer != null) {
         ExecutionTime = TimeSpan.Zero;
@@ -223 +252 @@
     }
 
-    public IEnumerable<IOptimizer> NestedOptimizers { get; }
+    public IEnumerable<IOptimizer> NestedOptimizers { get; private set; }
 
     public string Filename { get; set; }
@@ -367 +396 @@
       var remainingRows = problemData.TrainingIndices.Skip(calculatedRows);
 
-      if (gsoAction == GoalSeekingOptimizerAction.Stop) OnStopped();
-      else if (!remainingRows.Any()) OnStopped();
-      else if (gsoAction == GoalSeekingOptimizerAction.Pause) OnPaused();
-      else if (gsoAction == GoalSeekingOptimizerAction.Start) {
-        var row = remainingRows.First();
-        problem.Configure(problemData, row);
-        optimizer.Prepare();
-        optimizer.Start();
-      } else if (ExecutionState == ExecutionState.Started) {
-        OnPaused();
-      } else OnStopped();
+      // extract results from the optimizer 
+      GetResults();
+
+      if (gsoAction == GoalSeekingOptimizerAction.Stop)
+        OnStopped();
+      else if (!remainingRows.Any()) {
+        OnStopped();
+      } else switch (gsoAction) {
+          case GoalSeekingOptimizerAction.Pause:
+            OnPaused();
+            break;
+          case GoalSeekingOptimizerAction.Start:
+            currentRow = remainingRows.First();
+            problem.Configure(problemData, currentRow);
+            optimizer.Prepare();
+            optimizer.Start();
+            break;
+          default:
+            if (ExecutionState == ExecutionState.Started) {
+              OnPaused();
+            } else OnStopped();
+            break;
+        }
     }
     #endregion
+    #endregion
+
+    #region aggregate results
+    private void GetResults() {
+      if (Problem is MultiObjectiveGoalSeekingProblem) {
+        var m = (DoubleMatrix)optimizer.Results["Pareto Front Solutions"].Value;
+        AggregateMultiObjectiveResults(m);
+      } else if (Problem is SingleObjectiveGoalSeekingProblem) {
+        var m = (DoubleMatrix)optimizer.Results["Best Solution"].Value;
+        AggregateSingleObjectiveResults(m);
+      }
+    }
+
+    private void AggregateSingleObjectiveResults(DoubleMatrix solutions) {
+      var indices = solutions.ColumnNames.Select((x, i) => Tuple.Create(x, i)).ToDictionary(x => x.Item1, x => x.Item2);
+
+      var goals = Problem.Goals.Where(x => x.Active).ToList();
+      var inputs = Problem.Inputs.Where(x => x.Active).ToList();
+
+      if (Results == null)
+        Results = new ResultCollection();
+      // goals
+      if (!Results.ContainsKey("Goals"))
+        Results.Add(new Result("Goals", new ResultCollection()));
+      var goalResults = (ResultCollection)Results["Goals"].Value;
+      foreach (var goal in goals) {
+        var estimatedGoalName = goal.Name + " (estimated)";
+        if (!goalResults.ContainsKey(goal.Name)) {
+          goalResults.Add(new Result(goal.Name, new ScatterPlot(goal.Name, "")));
+        }
+
+        var plot = (ScatterPlot)goalResults[goal.Name].Value;
+        if (!plot.Rows.ContainsKey(goal.Name)) {
+          plot.Rows.Add(new ScatterPlotDataRow { Name = goal.Name });
+          plot.Rows.Add(new ScatterPlotDataRow { Name = estimatedGoalName });
+        }
+
+        plot.Rows[goal.Name].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[goal.Name]]));
+        plot.Rows[estimatedGoalName].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[estimatedGoalName]]));
+      }
+
+      // inputs
+      if (!Results.ContainsKey("Inputs"))
+        Results.Add(new Result("Inputs", new ResultCollection()));
+      var inputResults = (ResultCollection)Results["Inputs"].Value;
+      foreach (var input in inputs) {
+        var estimatedInputName = input.Name + " (estimated)";
+        if (!inputResults.ContainsKey(input.Name)) {
+          inputResults.Add(new Result(input.Name, new ScatterPlot(input.Name, "")));
+        }
+
+        var plot = (ScatterPlot)inputResults[input.Name].Value;
+        if (!plot.Rows.ContainsKey(input.Name)) {
+          plot.Rows.Add(new ScatterPlotDataRow { Name = input.Name });
+          plot.Rows.Add(new ScatterPlotDataRow { Name = estimatedInputName });
+        }
+
+        plot.Rows[input.Name].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[input.Name]]));
+        plot.Rows[estimatedInputName].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[estimatedInputName]]));
+      }
+      // aggregate results as a matrix
+      UpdateResultsMatrix(solutions);
+    }
+
+    private void AggregateMultiObjectiveResults(DoubleMatrix solutions) {
+      int maxSolutionsCount = Math.Min(solutions.Rows, 10);
+
+      var indices = solutions.ColumnNames.Select((x, i) => Tuple.Create(x, i)).ToDictionary(x => x.Item1, x => x.Item2);
+
+      var goals = Problem.Goals.Where(x => x.Active).ToList();
+      var inputs = Problem.Inputs.Where(x => x.Active).ToList();
+
+      if (Results == null)
+        Results = new ResultCollection();
+      // goals
+      if (!Results.ContainsKey("Goals"))
+        Results.Add(new Result("Goals", new ResultCollection()));
+      var goalResults = (ResultCollection)Results["Goals"].Value;
+      foreach (var goal in goals) {
+        var estimatedGoalName = goal.Name + " (estimated)";
+        if (!goalResults.ContainsKey(goal.Name)) {
+          goalResults.Add(new Result(goal.Name, new ScatterPlot(goal.Name, "")));
+        }
+
+        var plot = (ScatterPlot)goalResults[goal.Name].Value;
+        if (!plot.Rows.ContainsKey(goal.Name)) {
+          plot.Rows.Add(new ScatterPlotDataRow { Name = goal.Name, VisualProperties = { PointSize = 5 } });
+        }
+
+        plot.Rows[goal.Name].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[goal.Name]]));
+        for (int i = 0; i < maxSolutionsCount; ++i) {
+          if (!plot.Rows.ContainsKey(estimatedGoalName + i))
+            plot.Rows.Add(new ScatterPlotDataRow { Name = estimatedGoalName + i, VisualProperties = { PointSize = 5 } });
+          plot.Rows[estimatedGoalName + i].Points.Add(new Point2D<double>(currentRow, solutions[i, indices[estimatedGoalName]]));
+        }
+      }
+      // inputs
+      if (!Results.ContainsKey("Inputs"))
+        Results.Add(new Result("Inputs", new ResultCollection()));
+      var inputResults = (ResultCollection)Results["Inputs"].Value;
+      foreach (var input in inputs) {
+        var estimatedInputName = input.Name + " (estimated)";
+        if (!inputResults.ContainsKey(input.Name)) {
+          inputResults.Add(new Result(input.Name, new ScatterPlot(input.Name, "")));
+        }
+
+        var plot = (ScatterPlot)inputResults[input.Name].Value;
+        if (!plot.Rows.ContainsKey(input.Name)) {
+          plot.Rows.Add(new ScatterPlotDataRow { Name = input.Name, VisualProperties = { PointSize = 5 } });
+        }
+
+        plot.Rows[input.Name].Points.Add(new Point2D<double>(currentRow, solutions[0, indices[input.Name]])); // it stays the same for all the rows
+        for (int i = 0; i < maxSolutionsCount; ++i) {
+          if (!plot.Rows.ContainsKey(estimatedInputName + i))
+            plot.Rows.Add(new ScatterPlotDataRow { Name = estimatedInputName + i, VisualProperties = { PointSize = 5 } });
+          plot.Rows[estimatedInputName + i].Points.Add(new Point2D<double>(currentRow, solutions[i, indices[estimatedInputName]]));
+        }
+      }
+      // also add a matrix containing the whole solution data including the row
+      UpdateResultsMatrix(solutions);
+    }
+
+    private void UpdateResultsMatrix(DoubleMatrix solutions) {
+      // also add a matrix containing the whole solution data including the row
+      var rows = problemData.TrainingIndices.Take(calculatedRows).ToList();
+      var m = new DoubleMatrix(calculatedRows, solutions.Columns + 1) {
+        RowNames = rows.Select(x => x.ToString()),
+        ColumnNames = new[] { "Row" }.Concat(solutions.ColumnNames)
+      };
+      DoubleMatrix old = null;
+      if (Results.ContainsKey("Solutions"))
+        old = (DoubleMatrix)Results["Solutions"].Value;
+      else
+        Results.Add(new Result("Solutions", m));
+
+      if (old != null && calculatedRows > 0) {
+        for (int i = 0; i < old.Rows; ++i) {
+          for (int j = 0; j < old.Columns; ++j) {
+            m[i, j] = old[i, j];
+          }
+        }
+      }
+      m[rows.Count - 1, 0] = currentRow;
+      for (int i = 1; i < m.Columns; ++i) {
+        m[rows.Count - 1, i] = solutions[0, i - 1];
+      }
+      Results["Solutions"].Value = m;
+    }
     #endregion
   }

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/GoalSeekingProblem.csproj

@@ -25 +25 @@
     <Prefer32Bit>false</Prefer32Bit>
     <PlatformTarget>AnyCPU</PlatformTarget>
+    <LangVersion>4</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/MultiObjectiveGoalSeekingProblem.cs

@@ -39 +39 @@
   public sealed class MultiObjectiveGoalSeekingProblem : MultiObjectiveBasicProblem<RealVectorEncoding>, IGoalSeekingProblem {
     #region parameter names
-    private const string ModifiableDatasetParameterName = "Dataset";
     private const string InputsParameterName = "Inputs";
     private const string GoalsParameterName = "Goals";
@@ -151 +150 @@
     public MultiObjectiveGoalSeekingProblem() {
       dataset = new ModifiableDataset();
-      Parameters.Add(new ValueParameter<IDataset>(ModifiableDatasetParameterName, dataset) { Hidden = true });
       Parameters.Add(new ValueParameter<CheckedItemList<InputParameter>>(InputsParameterName));
       Parameters.Add(new ValueParameter<CheckedItemList<GoalParameter>>(GoalsParameterName));

branches/HeuristicLab.GoalSeekingProblem/HeuristicLab.GoalSeekingProblem/3.4/SingleObjectiveGoalSeekingProblem.cs

@@ -26 +26 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
 using HeuristicLab.Optimization;
@@ -38 +39 @@
   public sealed class SingleObjectiveGoalSeekingProblem : SingleObjectiveBasicProblem<RealVectorEncoding>, IGoalSeekingProblem {
     #region parameter names
-    private const string ModifiableDatasetParameterName = "Dataset";
     private const string InputsParameterName = "Inputs";
     private const string GoalsParameterName = "Goals";
@@ -136 +136 @@
     public SingleObjectiveGoalSeekingProblem() {
       dataset = new ModifiableDataset();
-      Parameters.Add(new ValueParameter<IDataset>(ModifiableDatasetParameterName, dataset) { Hidden = true });
       Parameters.Add(new ValueParameter<CheckedItemList<InputParameter>>(InputsParameterName));
       Parameters.Add(new ValueParameter<CheckedItemList<GoalParameter>>(GoalsParameterName));
@@ -161 +160 @@
       return quality;
     }
+
+    public override void Analyze(Individual[] individuals, double[] qualities, ResultCollection results, IRandom random) {
+      var zipped = individuals.Zip(qualities, (ind, qual) => new { Individual = ind, Quality = qual }).OrderBy(x => x.Quality);
+      var best = Maximization ? zipped.Last() : zipped.First();
+      var realVector = best.Individual.RealVector();
+      const string resultName = "Best Solution";
+
+      var columnNames = new List<string>();
+      foreach (var goal in ActiveGoals) {
+        columnNames.Add(goal.Name);
+        columnNames.Add(goal.Name + " (estimated)");
+      }
+      foreach (var input in ActiveInputs) {
+        columnNames.Add(input.Name);
+        columnNames.Add(input.Name + " (estimated)");
+        columnNames.Add(input.Name + " (deviation)");
+      }
+
+      var m = new DoubleMatrix(1, columnNames.Count) { ColumnNames = columnNames };
+      int i = 0;
+
+      var goals = ActiveGoals.Zip(GetEstimatedGoalValues(realVector, round: true),
+        (goal, value) => new { TargetValue = goal.Goal, EstimatedValue = value });
+
+      foreach (var goal in goals) {
+        m[0, i] = goal.TargetValue;
+        m[0, i + 1] = goal.EstimatedValue;
+        i += 2;
+      }
+
+      var inputs = ActiveInputs.Zip(realVector,
+        (input, value) => new { ActualValue = input.Value, EstimatedValue = value });
+
+      foreach (var input in inputs) {
+        m[0, i] = input.ActualValue;
+        m[0, i + 1] = input.EstimatedValue;
+        m[0, i + 2] = m[0, i] - m[0, i + 1];
+        i += 3;
+      }
+
+      if (!results.ContainsKey(resultName)) {
+        results.Add(new Result(resultName, m));
+      } else {
+        results[resultName].Value = m;
+      }
+
+      base.Analyze(individuals, qualities, results, random);
+    }
+
     #region event handlers
-
     private void RegisterEvents() {
       ModelsParameter.Value.ItemsAdded += ModelCollection_ItemsChanged;