Changeset 17776 for branches/3076_IA_evaluators_analyzers/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression

Timestamp:

10/19/20 14:14:11 (4 years ago)

Author:

dpiringe

Message:

#3076:

• changed SymbolicRegressionSingleObjectiveMetaModelAnalyzer to calculate the average quality of an solution for all problems
  • average: because we need to include outlier
• some architectural changes
• added new (but empty) meta model analyzer for multi objective -> SymbolicRegressionMultiObjectiveMetaModelAnalyzer

Location:

branches/3076_IA_evaluators_analyzers/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4

Files:

• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj (modified) (1 diff)
• SymbolicRegressionMetaModelAnalyzer.cs (modified) (6 diffs)
• SymbolicRegressionMultiObjectiveMetaModelAnalyzer.cs (added)
• SymbolicRegressionSingleObjectiveMetaModelAnalyzer.cs (modified) (3 diffs)

branches/3076_IA_evaluators_analyzers/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj

@@ -183 +183 @@
     <Compile Include="SymbolicRegressionConstraintAnalyzer.cs" />
     <Compile Include="SymbolicRegressionMetaModelAnalyzer.cs" />
+    <Compile Include="SymbolicRegressionMultiObjectiveMetaModelAnalyzer.cs" />
     <Compile Include="SymbolicRegressionPhenotypicDiversityAnalyzer.cs" />
     <Compile Include="SymbolicRegressionPruningAnalyzer.cs" />

branches/3076_IA_evaluators_analyzers/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SymbolicRegressionMetaModelAnalyzer.cs

@@ -22 +22 @@
     private const string BaseProblemParameterName = "Base Problem";
     private const string AnalyzeXIterationParameterName = "Analyze X Iteration";
-    private const string BestMetaModelParameterName = "Best Meta Model";
     #endregion
 
@@ -30 +29 @@
       (IFixedValueParameter<ItemList<T>>)Parameters[ProblemsToAnalyzeParameterName];
 
-    public IValueParameter<SymbolicRegressionSingleObjectiveProblem> BaseProblemParameter =>
-      (IValueParameter<SymbolicRegressionSingleObjectiveProblem>)Parameters[BaseProblemParameterName];
+    public IValueParameter<T> BaseProblemParameter =>
+      (IValueParameter<T>)Parameters[BaseProblemParameterName];
 
     public IFixedValueParameter<IntValue> AnalyzeXIterationParameter =>
       (IFixedValueParameter<IntValue>)Parameters[AnalyzeXIterationParameterName];
-
-    public IResultParameter<ISymbolicRegressionSolution> BestMetaModelParameter =>
-      (IResultParameter<ISymbolicRegressionSolution>)Parameters[BestMetaModelParameterName];
     #endregion
 
     public static int Iterations { get; set; } = 1;
 
+    #region constructors and cloning
     protected SymbolicRegressionMetaModelAnalyzer(SymbolicRegressionMetaModelAnalyzer<T> original, Cloner cloner) :
       base(original, cloner) { }
@@ -55 +52 @@
       Parameters.Add(new FixedValueParameter<IntValue>(AnalyzeXIterationParameterName,
         "After every X iteration, the analyzer will perform its step.", new IntValue(1)));
-      Parameters.Add(new ResultParameter<ISymbolicRegressionSolution>(BestMetaModelParameterName,
-        "The best meta model found."));
     }
 
@@ -72 +67 @@
         Parameters.Add(new FixedValueParameter<IntValue>(AnalyzeXIterationParameterName,
           "After every X iteration, the analyzer will perform its step.", new IntValue(1)));
+    }
 
-      if (!Parameters.ContainsKey(BestMetaModelParameterName))
-        Parameters.Add(new ResultParameter<ISymbolicRegressionSolution>(BestMetaModelParameterName,
-          "The best meta model found."));
-    }
+    #endregion
 
     public override void InitializeState() {
@@ -89 +82 @@
 
     //protected abstract bool TryFittingSolution(ISymbolicExpressionTree tree, T problem, out SymbolicRegressionSolution solution);
-    protected abstract void PerformApply(T problem, string targetVariable);
+    protected abstract void PerformApply(T baseProblem, IEnumerable<T> problems, string targetVariable);
 
     public override IOperation Apply() {
@@ -98 +91 @@
       if (Iterations >= AnalyzeXIterationParameter.Value.Value) {
         Iterations = 1;
-        string targetVariable = BaseProblemParameter.Value.ProblemData.TargetVariable;
+        var baseProblem = BaseProblemParameter.Value;
+        var problems = ProblemsToAnalyzeParameter.Value;
+        var targetVariable = BaseProblemParameter.Value.ProblemData.TargetVariable;
 
-        foreach (var problem in ProblemsToAnalyzeParameter.Value) {
-          if (problem.ProblemData.TargetVariable == targetVariable)
-            PerformApply(problem, targetVariable);
-          else
-            throw new ArgumentException($"The target variable of the problem '{problem.Name}' does not match with the base problem.");
-        }
-        
-        /*
-          * besser als eigenes Problem (abgeleitet von SymbolicRegressionProblem?), vlt auch als "CompoundProblem" (mehrere SymReg Problems in einen)?
-          * eigenes ProblemData?
-          * oder im Analyzer einfach anstatt Dataset -> ProblemData?
-          */
+        // error handling
+        var badProblems = problems.Where(x => x.ProblemData.TargetVariable != targetVariable);
+        IList<Exception> errors = new List<Exception>();
+        foreach (var problem in badProblems)
+          errors.Add(new ArgumentException($"The target variable of the problem '{problem.Name}' does not match with the base problem."));
+        if (badProblems.Any())
+          throw new AggregateException(errors);
+
+        // apply
+        PerformApply(baseProblem, problems, targetVariable);
       } else {
         Iterations++;

branches/3076_IA_evaluators_analyzers/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SymbolicRegressionSingleObjectiveMetaModelAnalyzer.cs

@@ -9 +9 @@
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -15 +16 @@
     : SymbolicRegressionMetaModelAnalyzer<SymbolicRegressionSingleObjectiveProblem>, ISymbolicExpressionTreeAnalyzer {
 
+    #region constants
+    private const string BestMetaModelParameterName = "Best Meta Model";
+    #endregion
+
+    #region parameter properties
+    public IResultParameter<ISymbolicRegressionSolution> BestMetaModelParameter =>
+      (IResultParameter<ISymbolicRegressionSolution>)Parameters[BestMetaModelParameterName];
+    #endregion
+
+    #region constructors and cloning
     [StorableConstructor]
     protected SymbolicRegressionSingleObjectiveMetaModelAnalyzer(StorableConstructorFlag _) : base(_) { }
@@ -20 +31 @@
     protected SymbolicRegressionSingleObjectiveMetaModelAnalyzer(SymbolicRegressionSingleObjectiveMetaModelAnalyzer original, Cloner cloner) : base(original, cloner) { }
 
-    public SymbolicRegressionSingleObjectiveMetaModelAnalyzer() { }
+    public SymbolicRegressionSingleObjectiveMetaModelAnalyzer() {
+      Parameters.Add(new ResultParameter<ISymbolicRegressionSolution>(BestMetaModelParameterName,
+        "The best meta model found."));
+    }
 
     [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(BestMetaModelParameterName))
+        Parameters.Add(new ResultParameter<ISymbolicRegressionSolution>(BestMetaModelParameterName,
+          "The best meta model found."));
+    }
 
     public override IDeepCloneable Clone(Cloner cloner) => new SymbolicRegressionSingleObjectiveMetaModelAnalyzer(this, cloner);
+    #endregion
 
-    protected override void PerformApply(SymbolicRegressionSingleObjectiveProblem problem, string targetVariable) {
-      double bestQuality = problem.Maximization.Value ? double.MinValue : double.MaxValue;
+    protected override void PerformApply(
+      SymbolicRegressionSingleObjectiveProblem baseProblem, 
+      IEnumerable<SymbolicRegressionSingleObjectiveProblem> problems, 
+      string targetVariable) {
+      // init
+      var solutions = this.SymbolicExpressionTree.ToArray();
+      var evaluator = baseProblem.Evaluator;
+      var bestQuality = baseProblem.Maximization.Value ? double.MinValue : double.MaxValue;
       SymbolicRegressionSolution bestMetaModel = null;
-      foreach (var tree in this.SymbolicExpressionTree.ToArray()) {
-        IDataset dataset = problem.ProblemData.Dataset;
-        IEnumerable<int> rows = Enumerable.Range(0, dataset.Rows);
-        var quality = problem.Evaluator.Evaluate(ExecutionContext, tree, problem.ProblemData, rows);
-        
-        bool isBetter = problem.Maximization.Value ? (bestQuality < quality) : (bestQuality > quality);
-        if(isBetter) {
-          bestQuality = quality;
-          var model = new SymbolicRegressionModel(
-            targetVariable, 
-            (ISymbolicExpressionTree)tree.Clone(), 
-            new SymbolicDataAnalysisExpressionTreeInterpreter());
-          bestMetaModel = new SymbolicRegressionSolution(model, problem.ProblemData);
+
+      // iterate solutions
+      foreach (var solution in solutions) {
+        double qualityAvg = CalculateAverageQuality(solution, evaluator, problems);
+
+        // check if this solution is the best
+        bool isBest = baseProblem.Maximization.Value ? (bestQuality < qualityAvg) : (bestQuality > qualityAvg);
+        if (isBest) {
+          bestQuality = qualityAvg;
+          bestMetaModel = BuildSolution(solution, targetVariable, baseProblem);
         }
       }
       BestMetaModelParameter.ActualValue = bestMetaModel;
     }
+
+    private SymbolicRegressionSolution BuildSolution(
+      ISymbolicExpressionTree solution, 
+      string targetVariable,
+      SymbolicRegressionSingleObjectiveProblem baseProblem) {
+      var model = new SymbolicRegressionModel(
+            targetVariable,
+            (ISymbolicExpressionTree)solution.Clone(),
+            new SymbolicDataAnalysisExpressionTreeInterpreter());
+      return new SymbolicRegressionSolution(model, baseProblem.ProblemData);
+    }
+
+    private double CalculateAverageQuality(
+      ISymbolicExpressionTree solution,
+      ISymbolicDataAnalysisSingleObjectiveEvaluator<IRegressionProblemData> evaluator, 
+      IEnumerable<SymbolicRegressionSingleObjectiveProblem> problems) {
+      double qualitySum = 0.0;
+      // iterate problems
+      foreach (var problem in problems) {
+        IDataset dataset = problem.ProblemData.Dataset;
+        IEnumerable<int> rows = Enumerable.Range(0, dataset.Rows);
+        // evalute problem with the evaluator of the base problem
+        qualitySum += evaluator.Evaluate(ExecutionContext, solution, problem.ProblemData, rows);
+      }
+      // calculate the average quality
+      return qualitySum / problems.Count();
+    }
   }
 }