Changeset 16461 for branches/2974_Constants_Optimization

Timestamp:

12/28/18 15:44:31 (6 years ago)

Author:

mkommend

Message:

#2974: Added unit tests and refactoring.

Location:

branches/2974_Constants_Optimization

Files:

• ConstantsOptimization.sln (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/ConstantsOptimizationEvaluator.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Constants Optimization/Util.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToAutoDiffTermConverter.cs (modified) (2 diffs)
• UnitTests (added)
• UnitTests/ConstantsOptimizationTests.cs (added)
• UnitTests/PerformanceTest.cs (added)
• UnitTests/Properties (added)
• UnitTests/Properties/AssemblyInfo.cs (added)
• UnitTests/UnitTests.csproj (added)
• UnitTests/packages.config (added)

branches/2974_Constants_Optimization/ConstantsOptimization.sln

@@ -15 +15 @@
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Problems.DataAnalysis.Symbolic-3.4", "HeuristicLab.Problems.DataAnalysis.Symbolic\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj", "{3D28463F-EC96-4D82-AFEE-38BE91A0CA00}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "UnitTests\UnitTests.csproj", "{9617D7C7-90FB-41BB-A9B2-6DAC68E15197}"
 EndProject
 Global
@@ -50 +52 @@
     {3D28463F-EC96-4D82-AFEE-38BE91A0CA00}.Release|x86.ActiveCfg = Release|x86
     {3D28463F-EC96-4D82-AFEE-38BE91A0CA00}.Release|x86.Build.0 = Release|x86
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|Any CPU.Build.0 = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|x64.ActiveCfg = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|x64.Build.0 = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|x86.ActiveCfg = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Debug|x86.Build.0 = Debug|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|Any CPU.ActiveCfg = Release|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|Any CPU.Build.0 = Release|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|x64.ActiveCfg = Release|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|x64.Build.0 = Release|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|x86.ActiveCfg = Release|Any CPU
+    {9617D7C7-90FB-41BB-A9B2-6DAC68E15197}.Release|x86.Build.0 = Release|Any CPU
   EndGlobalSection
   GlobalSection(SolutionProperties) = preSolution

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj

@@ -102 +102 @@
       <HintPath>..\..\..\..\trunk\bin\ALGLIB-3.7.0.dll</HintPath>
       <Private>False</Private>
+    </Reference>
+    <Reference Include="AutoDiff-1.0, Version=1.0.0.14388, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\..\trunk\bin\AutoDiff-1.0.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Analysis-3.3">

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/ConstantsOptimizationEvaluator.cs

@@ -131 +131 @@
     }
 
-
     private static readonly object locker = new object();
     public override IOperation InstrumentedApply() {
@@ -189 +188 @@
     }
 
+    [ThreadStatic]
+    private static double[,] x = null;
+    [ThreadStatic]
+    private static IDataset ds = null;
+    [ThreadStatic]
+    private static Dictionary<DataForVariable, AutoDiff.Variable> parameters;
+
     public static double OptimizeConstants(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
       ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling,
@@ -195 +201 @@
       bool updateConstantsInTree = true, Action<double[], double, object> iterationCallback = null, EvaluationsCounter counter = null) {
 
+      if (ds == null || ds != problemData.Dataset) {
+        ds = problemData.Dataset;
+        parameters = ConstantsOptimization.Util.ExtractParameters(problemData.Dataset);
+        x = ConstantsOptimization.Util.ExtractData(ds, parameters.Keys, rows);
+        
+      }
+      double[] y = ds.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
+
 
       // numeric constants in the tree become variables for constant opt
@@ -201 +215 @@
       // variable name, variable value (for factor vars) and lag as a DataForVariable object.
       // A dictionary is used to find parameters
-      double[] initialConstants;
-      var parameters = new List<TreeToAutoDiffTermConverter.DataForVariable>();
-
       TreeToAutoDiffTermConverter.ParametricFunction func;
       TreeToAutoDiffTermConverter.ParametricFunctionGradient func_grad;
-      if (!TreeToAutoDiffTermConverter.TryConvertToAutoDiff(tree, updateVariableWeights, applyLinearScaling, out parameters, out initialConstants, out func, out func_grad))
+
+      if (!TreeToAutoDiffTermConverter.TryConvertToAutoDiff(tree, updateVariableWeights, applyLinearScaling, parameters, out func, out func_grad))
         throw new NotSupportedException("Could not optimize constants of symbolic expression tree due to not supported symbols used in the tree.");
       if (parameters.Count == 0) return 0.0; // gkronber: constant expressions always have a R² of 0.0 
-      //var parameterEntries = parameters.ToArray(); // order of entries must be the same for x
+
+      var initialConstants = ConstantsOptimization.Util.ExtractConstants(tree);
 
       //extract inital constants
@@ -231 +244 @@
       int retVal;
 
-      IDataset ds = problemData.Dataset;
-      var x = ConstantsOptimization.Util.ExtractData(ds, parameters, rows);
-      double[] y = ds.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
+
       int n = x.GetLength(0);
       int m = x.GetLength(1);
@@ -263 +274 @@
           var tmp = new double[c.Length - 2];
           Array.Copy(c, 2, tmp, 0, tmp.Length);
-          UpdateConstants(tree, tmp, updateVariableWeights);
-        } else UpdateConstants(tree, c, updateVariableWeights);
+          ConstantsOptimization.Util.UpdateConstants(tree, tmp);
+        } else ConstantsOptimization.Util.UpdateConstants(tree, c);
       }
       var quality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling);
 
-      if (!updateConstantsInTree) UpdateConstants(tree, initialConstants, updateVariableWeights);
+      if (!updateConstantsInTree)
+        ConstantsOptimization.Util.UpdateConstants(tree, initialConstants);
 
       if (originalQuality - quality > 0.001 || double.IsNaN(quality)) {
-        UpdateConstants(tree, initialConstants, updateVariableWeights);
+        ConstantsOptimization.Util.UpdateConstants(tree, initialConstants);
         return originalQuality;
       }
       return quality;
-    }
-
-    private static void UpdateConstants(ISymbolicExpressionTree tree, double[] constants, bool updateVariableWeights) {
-      int i = 0;
-      foreach (var node in tree.Root.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>()) {
-        ConstantTreeNode constantTreeNode = node as ConstantTreeNode;
-        VariableTreeNodeBase variableTreeNodeBase = node as VariableTreeNodeBase;
-        FactorVariableTreeNode factorVarTreeNode = node as FactorVariableTreeNode;
-        if (constantTreeNode != null)
-          constantTreeNode.Value = constants[i++];
-        else if (updateVariableWeights && variableTreeNodeBase != null)
-          variableTreeNodeBase.Weight = constants[i++];
-        else if (factorVarTreeNode != null) {
-          for (int j = 0; j < factorVarTreeNode.Weights.Length; j++)
-            factorVarTreeNode.Weights[j] = constants[i++];
-        }
-      }
     }
 

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Constants Optimization/Util.cs

@@ -37 +37 @@
         foreach (var variable in variables) {
           if (dataset.VariableHasType<double>(variable.variableName)) {
-            x[row, col] = dataset.GetDoubleValue(variable.variableName, r+variable.lag);
+            x[row, col] = dataset.GetDoubleValue(variable.variableName, r + variable.lag);
           } else if (dataset.VariableHasType<string>(variable.variableName)) {
             x[row, col] = dataset.GetStringValue(variable.variableName, r) == variable.variableValue ? 1 : 0;
@@ -50 +50 @@
     public static Dictionary<DataForVariable, AutoDiff.Variable> ExtractParameters(IDataset dataset) {
       var parameters = new Dictionary<DataForVariable, AutoDiff.Variable>();
-      foreach(var doubleVariable in dataset.DoubleVariables) {
+      foreach (var doubleVariable in dataset.DoubleVariables) {
         var data = new DataForVariable(doubleVariable, string.Empty, 0);
         var param = new AutoDiff.Variable();
-        parameters.Add(data,param);
+        parameters.Add(data, param);
       }
 
-      foreach(var stringVariable in dataset.StringVariables) {
-        foreach(var stringValue in dataset.GetStringValues(stringVariable).Distinct()) {
-          var data = new DataForVariable(stringVariable, stringVariable, 0);
+      foreach (var stringVariable in dataset.StringVariables) {
+        foreach (var stringValue in dataset.GetStringValues(stringVariable).Distinct()) {
+          var data = new DataForVariable(stringVariable, stringValue, 0);
           var param = new AutoDiff.Variable();
           parameters.Add(data, param);
@@ -64 +64 @@
       }
       return parameters;
+    }
+
+    public static double[] ExtractConstants(ISymbolicExpressionTree tree) {
+      var constants = new List<double>();
+      foreach (var node in tree.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>()) {
+        ConstantTreeNode constantTreeNode = node as ConstantTreeNode;
+        VariableTreeNodeBase variableTreeNodeBase = node as VariableTreeNodeBase;
+        FactorVariableTreeNode factorVarTreeNode = node as FactorVariableTreeNode;
+        if (constantTreeNode != null)
+          constants.Add(constantTreeNode.Value);
+        else if (variableTreeNodeBase != null)
+          constants.Add(variableTreeNodeBase.Weight);
+        else if (factorVarTreeNode != null) {
+          for (int j = 0; j < factorVarTreeNode.Weights.Length; j++)
+            constants.Add(factorVarTreeNode.Weights[j]);
+        }        
+      }
+      return constants.ToArray();
     }
 

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToAutoDiffTermConverter.cs

@@ -131 +131 @@
     }
 
+    public static bool TryConvertToAutoDiff(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms, Dictionary<DataForVariable, AutoDiff.Variable> parameters,
+      out ParametricFunction func,
+      out ParametricFunctionGradient func_grad
+  ) {
+
+      // use a transformator object which holds the state (variable list, parameter list, ...) for recursive transformation of the tree
+      var transformator = new TreeToAutoDiffTermConverter(makeVariableWeightsVariable, parameters);
+      AutoDiff.Term term;
+      try {
+        term = transformator.ConvertToAutoDiff(tree.Root.GetSubtree(0));
+
+        if (addLinearScalingTerms) {
+          // scaling variables α, β are given at the beginning of the parameter vector
+          var alpha = new AutoDiff.Variable();
+          var beta = new AutoDiff.Variable();
+          transformator.variables.Insert(0, alpha);
+          transformator.variables.Insert(0, beta);
+
+          term = term * alpha + beta;
+        }
+        
+        var compiledTerm = term.Compile(transformator.variables.ToArray(), parameters.Select(kvp => kvp.Value).ToArray());
+
+
+        func = (vars, @params) => compiledTerm.Evaluate(vars, @params);
+        func_grad = (vars, @params) => compiledTerm.Differentiate(vars, @params);
+        return true;
+      } catch (ConversionException) {
+        func = null;
+        func_grad = null;
+      }
+      return false;
+    }
+
     // state for recursive transformation of trees 
     private readonly List<double> initialConstants;
@@ -137 +171 @@
     private readonly bool makeVariableWeightsVariable;
 
-    private TreeToAutoDiffTermConverter(bool makeVariableWeightsVariable) {
+    private TreeToAutoDiffTermConverter(bool makeVariableWeightsVariable, Dictionary<DataForVariable, AutoDiff.Variable> parameters = null) {
       this.makeVariableWeightsVariable = makeVariableWeightsVariable;
       this.initialConstants = new List<double>();
-      this.parameters = new Dictionary<DataForVariable, AutoDiff.Variable>();
+      if (parameters == null)
+        this.parameters = new Dictionary<DataForVariable, AutoDiff.Variable>();
+      else
+        this.parameters = parameters;
       this.variables = new List<AutoDiff.Variable>();
     }