Changeset 18213 for branches/3138_Shape_Constraints_Transformations/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression

Timestamp:

02/08/22 13:06:49 (2 years ago)

Author:

dpiringe

Message:

#3138

• reimplemented extended constraints with an interface and an implementation, because of project reference troubles
• moved the extended shape constraints into IShapeConstrainedRegressionProblemData
• added thresholds for shape constraints -> the error is linear between 0 and 1, error caps at 1 when error >= (intervalbound + thresholdbound)
  • adapted ShapeConstraint and ShapeConstraintsParser to identify and store thresholds
• adapted IntervalUtil to work with thresholds
• adapted NMSESingleObjectiveConstraintsEvaluator and ShapeConstraintsAnalyzer to work with extended constraints
• added a new chart in ShapeConstraintsAnalyzer to show the average constraint violation

Location:

branches/3138_Shape_Constraints_Transformations/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4

Files:

• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj (modified) (1 diff)
• ShapeConstraintsAnalyzer.cs (modified) (7 diffs)
• SingleObjective/Evaluators/NMSESingleObjectiveConstraintsEvaluator.cs (modified) (11 diffs)

branches/3138_Shape_Constraints_Transformations/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj

@@ -263 +263 @@
       <Private>False</Private>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>

branches/3138_Shape_Constraints_Transformations/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/ShapeConstraintsAnalyzer.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections;
+using System.Collections.Generic;
 using System.Linq;
 using HEAL.Attic;
@@ -28 +30 @@
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -36 +39 @@
     private const string ConstraintViolationsParameterName = "ConstraintViolations";
     private const string InfeasibleSolutionsParameterName = "InfeasibleSolutions";
+    private const string AverageConstraintViolationsParameterName = "AverageConstraintViolations";
+    private const string SymbolicDataAnalysisTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
 
     #region parameter properties
@@ -48 +53 @@
       (IResultParameter<DataTable>)Parameters[InfeasibleSolutionsParameterName];
 
+    public IResultParameter<DataTable> AverageConstraintViolationsParameter =>
+      (IResultParameter<DataTable>)Parameters[AverageConstraintViolationsParameterName];
+
+    public ILookupParameter<ISymbolicDataAnalysisExpressionTreeInterpreter> SymbolicDataAnalysisTreeInterpreterParameter =>
+      (ILookupParameter<ISymbolicDataAnalysisExpressionTreeInterpreter>)Parameters[SymbolicDataAnalysisTreeInterpreterParameterName];
+    
     #endregion
 
-    #region properties
+      #region properties
     public IRegressionProblemData RegressionProblemData => RegressionProblemDataParameter.ActualValue;
     public DataTable ConstraintViolations => ConstraintViolationsParameter.ActualValue;
     public DataTable InfeasibleSolutions => InfeasibleSolutionsParameter.ActualValue;
+    public DataTable AverageConstraintViolations => AverageConstraintViolationsParameter.ActualValue;
     #endregion
 
@@ -75 +87 @@
       Parameters.Add(new ResultParameter<DataTable>(InfeasibleSolutionsParameterName,
         "The number of infeasible solutions."));
+      Parameters.Add(new ResultParameter<DataTable>(AverageConstraintViolationsParameterName,
+        "The average violations of each constraint."));
+      Parameters.Add(new LookupParameter<ISymbolicDataAnalysisExpressionTreeInterpreter>(SymbolicDataAnalysisTreeInterpreterParameterName, 
+        "The interpreter that should be used to calculate the output values of the symbolic data analysis tree.") { Hidden = true });
+
 
 
@@ -90 +107 @@
         }
       };
+
+      AverageConstraintViolationsParameter.DefaultValue = new DataTable(SymbolicDataAnalysisTreeInterpreterParameterName) {
+        VisualProperties = {
+          XAxisTitle = "Generations",
+          YAxisTitle = "Average Constraint Violations"
+        }
+      };
     }
 
 
     [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(SymbolicDataAnalysisTreeInterpreterParameterName))
+        Parameters.Add(new LookupParameter<ISymbolicDataAnalysisExpressionTreeInterpreter>(SymbolicDataAnalysisTreeInterpreterParameterName,
+          "The interpreter that should be used to calculate the output values of the symbolic data analysis tree.") { Hidden = true });
+    }
 
     public override IOperation Apply() {
       var problemData = (IShapeConstrainedRegressionProblemData)RegressionProblemData;
       var trees = SymbolicExpressionTree.ToArray();
-
+      
       var results = ResultCollection;
-      var constraints = problemData.ShapeConstraints.EnabledConstraints;
+      var modelConstraints = problemData.ShapeConstraints.EnabledConstraints;
+      var extendedConstraints = problemData.CheckedExtendedConstraints;
       var variableRanges = problemData.VariableRanges;
       var constraintViolationsTable = ConstraintViolations;
+      var averageConstraintViolations = AverageConstraintViolations;
+      var interpreter = SymbolicDataAnalysisTreeInterpreterParameter.ActualValue;
       var estimator = new IntervalArithBoundsEstimator();
+      
+      if (!constraintViolationsTable.Rows.Any()) {
+        foreach (var constraint in modelConstraints) {
+          constraintViolationsTable.Rows.Add(new DataRow(constraint.ToString()));
+          averageConstraintViolations.Rows.Add(new DataRow(constraint.ToString()));
+        }
 
-      if (!constraintViolationsTable.Rows.Any())
-        foreach (var constraint in constraints)
-          constraintViolationsTable.Rows.Add(new DataRow(constraint.ToString()));
+        foreach (var extendedConstraint in extendedConstraints.SelectMany(x => x.ShapeConstraints.EnabledConstraints)) {
+          constraintViolationsTable.Rows.Add(new DataRow(extendedConstraint.ToString()));
+          averageConstraintViolations.Rows.Add(new DataRow(extendedConstraint.ToString()));
+        }
+      }
 
-      foreach (var constraint in constraints) {
-        var numViolations = trees.Count(tree => IntervalUtil.GetConstraintViolation(constraint, estimator, variableRanges, tree) > 0.0);
-        constraintViolationsTable.Rows[constraint.ToString()].Values.Add(numViolations);
+      var violationsPerTree = new Dictionary<ISymbolicExpressionTree, int>();
+      var violationsPerConstraint = new Dictionary<string, IList<double>>();
+
+      foreach(var tree in trees) {
+        var violations = NMSESingleObjectiveConstraintsEvaluator.CalculateShapeConstraintsViolations(problemData, tree, interpreter, estimator, new MersenneTwister());
+        foreach(var violation in violations) {
+          var constraint = violation.Item1;
+          var error = violation.Item2;
+          if (!violationsPerConstraint.ContainsKey(constraint.ToString()))
+            violationsPerConstraint.Add(constraint.ToString(), new List<double>());
+          violationsPerConstraint[constraint.ToString()].Add(error);
+        }
+        violationsPerTree.Add(tree, violations.Count(x => x.Item2 > 0));
+      }
+
+      foreach (var constraint in modelConstraints) {
+        var errors = violationsPerConstraint[constraint.ToString()];
+        constraintViolationsTable.Rows[constraint.ToString()].Values.Add(errors.Count(x => x > 0));
+        averageConstraintViolations.Rows[constraint.ToString()].Values.Add(errors.Sum() / errors.Count());
+      }
+
+      foreach (var extendedConstraint in extendedConstraints.SelectMany(x => x.ShapeConstraints.EnabledConstraints)) {
+        var errors = violationsPerConstraint[extendedConstraint.ToString()];
+        constraintViolationsTable.Rows[extendedConstraint.ToString()].Values.Add(errors.Count(x => x > 0));
+        averageConstraintViolations.Rows[extendedConstraint.ToString()].Values.Add(errors.Sum() / errors.Count());
       }
 
@@ -121 +182 @@
       infeasibleSolutionsDataTable.Rows[InfeasibleSolutionsParameterName]
         .Values
-        .Add(trees.Count(t => IntervalUtil.GetConstraintViolations(constraints, estimator, variableRanges, t).Any(x => x > 0.0)));
+        .Add(trees.Count(t => violationsPerTree[t] > 0));
 
       return base.Apply();

branches/3138_Shape_Constraints_Transformations/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/NMSESingleObjectiveConstraintsEvaluator.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -29 +30 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Parameters;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -41 +43 @@
     private const string BoundsEstimatorParameterName = "BoundsEstimator";
     private const string PenaltyFactorParameterName = "PenaltyFactor";
-    private const string ExtendedConstraintsParameterName = "ExtendedConstraints";
 
 
@@ -58 +59 @@
       (IFixedValueParameter<DoubleValue>)Parameters[PenaltyFactorParameterName];
 
-    public IFixedValueParameter<IItemList<ExtendedConstraint>> ExtendedConstraintsParameter =>
-      (IFixedValueParameter<IItemList<ExtendedConstraint>>)Parameters[ExtendedConstraintsParameterName];
 
 
@@ -86 +85 @@
       set => PenaltyFactorParameter.Value.Value = value;
     }
-
-    public IEnumerable<ExtendedConstraint> ExtendedConstraints {
-      get => ExtendedConstraintsParameter.Value;
-    }
-      
-
 
     public override bool Maximization => false; // NMSE is minimized
@@ -116 +109 @@
       Parameters.Add(new FixedValueParameter<DoubleValue>(PenaltyFactorParameterName,
         "Punishment factor for constraint violations for soft constraint handling (fitness = NMSE + penaltyFactor * avg(violations)) (default: 1.0)", new DoubleValue(1.0)));
-      Parameters.Add(new FixedValueParameter<ItemList<ExtendedConstraint>>(ExtendedConstraintsParameterName, "", new ItemList<ExtendedConstraint>()));
     }
 
@@ -135 +127 @@
       var estimationLimits = EstimationLimitsParameter.ActualValue;
       var applyLinearScaling = ApplyLinearScalingParameter.ActualValue.Value;
+      var random = RandomParameter.ActualValue;
 
       if (OptimizeParameters) {
@@ -175 +168 @@
 
       var quality = Calculate(interpreter, tree, estimationLimits.Lower, estimationLimits.Upper, problemData, rows,
-        BoundsEstimator, UseSoftConstraints, PenalityFactor, ExtendedConstraints);
+        BoundsEstimator, random, UseSoftConstraints, PenalityFactor);
       QualityParameter.ActualValue = new DoubleValue(quality);
 
@@ -186 +179 @@
       double lowerEstimationLimit, double upperEstimationLimit,
       IRegressionProblemData problemData, IEnumerable<int> rows,
-      IBoundsEstimator estimator,
-      bool useSoftConstraints = false, double penaltyFactor = 1.0,
-      IEnumerable<ExtendedConstraint> extendedConstraints = null) {
-
-      var estimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows);
+      IBoundsEstimator estimator, IRandom random,
+      bool useSoftConstraints = false, double penaltyFactor = 1.0) {
+
+      var trainingEstimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows);
       var targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
-      var constraints = Enumerable.Empty<ShapeConstraint>();
+
+      var trainingBoundedEstimatedValues = trainingEstimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit);
+      var nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(targetValues, trainingBoundedEstimatedValues,
+        out var errorState);
+
+      if (errorState != OnlineCalculatorError.None)
+        return double.MaxValue;
+
+      var violations = Enumerable.Empty<double>();
       if (problemData is ShapeConstrainedRegressionProblemData scProbData) {
-        constraints = scProbData.ShapeConstraints.EnabledConstraints;
-      } 
-      var intervalCollection = problemData.VariableRanges;
-
-      var boundedEstimatedValues = estimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit);
-      var nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(targetValues, boundedEstimatedValues,
-        out var errorState);
-
-      if (errorState != OnlineCalculatorError.None) {
-        return 1.0;
-      }
-
-      var constraintViolations = IntervalUtil.GetConstraintViolations(constraints, estimator, intervalCollection, tree);
-
-      if (constraintViolations.Any(x => double.IsNaN(x) || double.IsInfinity(x))) {
-        return 1.0;
-      }
+        violations = CalculateShapeConstraintsViolations(scProbData, tree, interpreter, estimator, random).Select(x => x.Item2);
+      }
+
+      if (violations.Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        return double.MaxValue;
 
       if (useSoftConstraints) {
@@ -216 +204 @@
           throw new ArgumentException("The parameter has to be >= 0.0.", nameof(penaltyFactor));
 
-        var weightedViolationsAvg = constraints
-          .Zip(constraintViolations, (c, v) => c.Weight * v)
-          .Average();
-
-        return Math.Min(nmse, 1.0) + penaltyFactor * weightedViolationsAvg;
-      } else if (constraintViolations.Any(x => x > 0.0)) {
-        return 1.0;
-      }
-
-      return nmse;
+        return nmse + penaltyFactor * violations.Average();
+      }
+      return violations.Any(x => x > 0.0) ? 1.0 : nmse;
+    }
+
+    public static IEnumerable<Tuple<ShapeConstraint, double>> CalculateShapeConstraintsViolations(
+      IShapeConstrainedRegressionProblemData problemData, ISymbolicExpressionTree tree, 
+      ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, IBoundsEstimator estimator,
+      IRandom random) {
+      IList<Tuple<ShapeConstraint, double>> violations = new List<Tuple<ShapeConstraint, double>>();
+      
+      var baseConstraints = problemData.ShapeConstraints.EnabledConstraints;
+      var intervalCollection = problemData.VariableRanges;
+      var extendedShapeConstraints = problemData.CheckedExtendedConstraints;
+      var allEstimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, problemData.AllIndices);
+
+      foreach (var constraint in baseConstraints)
+        violations.Add(Tuple.Create(constraint, IntervalUtil.GetConstraintViolation(constraint, estimator, intervalCollection, tree) * constraint.Weight));
+
+      IDictionary<string, IList> dict = new Dictionary<string, IList>();
+      foreach (var varName in problemData.Dataset.VariableNames) {
+        if (varName != problemData.TargetVariable)
+          dict.Add(varName, problemData.Dataset.GetDoubleValues(varName).ToList());
+        else dict.Add(varName, allEstimatedValues.ToList());
+      }
+      var tmpDataset = new Dataset(dict.Keys, dict.Values);
+
+      foreach (var extendedConstraint in extendedShapeConstraints) {
+        var enabledConstraints = extendedConstraint.ShapeConstraints.EnabledConstraints;
+        if (enabledConstraints.Any()) {
+          var extendedConstraintExprValues = interpreter.GetSymbolicExpressionTreeValues(extendedConstraint.Tree, tmpDataset, problemData.AllIndices);
+          var extendedConstraintExprInterval = new Interval(extendedConstraintExprValues.Min(), extendedConstraintExprValues.Max());
+
+          foreach (var constraint in enabledConstraints) {
+            if (constraint.Regions.Count > 0) {
+              // adapt dataset
+              foreach (var kvp in constraint.Regions.GetReadonlyDictionary()) {
+                var lb = double.IsNegativeInfinity(kvp.Value.LowerBound) ? double.MinValue : kvp.Value.LowerBound;
+                var ub = double.IsPositiveInfinity(kvp.Value.UpperBound) ? double.MaxValue : kvp.Value.UpperBound;
+
+                var vals = Enumerable.Range(0, dict[kvp.Key].Count - 2)
+                                     .Select(x => UniformDistributedRandom.NextDouble(random, lb, ub))
+                                     .ToList();
+                vals.Add(lb);
+                vals.Add(ub);
+                vals.Sort();
+                dict[kvp.Key] = vals;
+              }
+              // calc again with new regions
+              tmpDataset = new Dataset(dict.Keys, dict.Values);
+              // calc target again
+              allEstimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, tmpDataset, problemData.AllIndices);
+              dict[problemData.TargetVariable] = allEstimatedValues.ToList();
+              tmpDataset = new Dataset(dict.Keys, dict.Values);
+              extendedConstraintExprValues = interpreter.GetSymbolicExpressionTreeValues(extendedConstraint.Tree, tmpDataset, problemData.AllIndices);
+              extendedConstraintExprInterval = new Interval(extendedConstraintExprValues.Min(), extendedConstraintExprValues.Max());
+            }
+            violations.Add(Tuple.Create(constraint, IntervalUtil.GetIntervalError(constraint.Interval, extendedConstraintExprInterval, constraint.Threshold) * constraint.Weight));
+          }
+        }
+      }
+      return violations;
     }
 
@@ -234 +274 @@
       EstimationLimitsParameter.ExecutionContext = context;
       ApplyLinearScalingParameter.ExecutionContext = context;
+      RandomParameter.ExecutionContext = context;
 
       var nmse = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree,
         EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper,
-        problemData, rows, BoundsEstimator, UseSoftConstraints, PenalityFactor, ExtendedConstraints);
+        problemData, rows, BoundsEstimator, RandomParameter.Value, UseSoftConstraints, PenalityFactor);
 
       SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
       EstimationLimitsParameter.ExecutionContext = null;
       ApplyLinearScalingParameter.ExecutionContext = null;
+      RandomParameter.ExecutionContext = null;
 
       return nmse;