Changeset 17721 for branches

Timestamp:

08/11/20 13:39:48 (4 years ago)

Author:

pfleck

Message:

#3040 First draft of different-vector-length strategies (cut, fill, resample, cycle, ...)

Location:

branches/3040_VectorBasedGP

Files:

• HeuristicLab.Common/3.3/Statistics/EnumerableStatisticExtensions.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeVectorInterpreter.cs (modified) (20 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VectorData/AzzaliBenchmark3.cs (modified) (1 diff)

branches/3040_VectorBasedGP/HeuristicLab.Common/3.3/Statistics/EnumerableStatisticExtensions.cs

@@ -230 +230 @@
       if (min > max) throw new ArgumentException(string.Format("Minimum {0} is larger than maximum {1}.", min, max));
       foreach (var x in values) {
-        if (double.IsNaN(x)) yield return (max + min) / 2.0;
-        else if (x < min) yield return min;
+        /*if (double.IsNaN(x)) yield return (max + min) / 2.0;
+        else */
+        if (x < min) yield return min;
         else if (x > max) yield return max;
         else yield return x;

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs

@@ -56 +56 @@
     private static readonly TF_DataType DataType = tf.float32;
 
-#region Parameter Properties
+    #region Parameter Properties
     public IFixedValueParameter<IntValue> ConstantOptimizationIterationsParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[MaximumIterationsName]; }
@@ -63 +63 @@
       get { return (IFixedValueParameter<DoubleValue>)Parameters[LearningRateName]; }
     }
-#endregion
-
-#region Properties
+    #endregion
+
+    #region Properties
     public int ConstantOptimizationIterations {
       get { return ConstantOptimizationIterationsParameter.Value.Value; }
@@ -72 +72 @@
       get { return LearningRateParameter.Value.Value; }
     }
-#endregion
+    #endregion
 
     public TensorFlowConstantOptimizationEvaluator()
@@ -125 +125 @@
       var target = tf.placeholder(DataType, new TensorShape(numRows), name: problemData.TargetVariable);
       // MSE
-      var cost = tf.reduce_mean(tf.square(prediction - target));
+      var cost = tf.reduce_mean(tf.square(target - prediction));
 
       var optimizer = tf.train.AdamOptimizer((float)learningRate);
@@ -144 +144 @@
         if (problemData.Dataset.VariableHasType<double>(variableName)) {
           var data = problemData.Dataset.GetDoubleValues(variableName, rows).Select(x => (float)x).ToArray();
-          variablesFeed.Add(variable, np.array(data, copy: false).reshape(numRows, 1));
+          variablesFeed.Add(variable, np.array(data).reshape(numRows, 1));
         } else if (problemData.Dataset.VariableHasType<DoubleVector>(variableName)) {
           var data = problemData.Dataset.GetDoubleVectorValues(variableName, rows).Select(x => x.Select(y => (float)y).ToArray()).ToArray();
@@ -152 +152 @@
       }
       var targetData = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).Select(x => (float)x).ToArray();
-      variablesFeed.Add(target, np.array(targetData, copy: false));
+      variablesFeed.Add(target, np.array(targetData));
 
 

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeVectorInterpreter.cs

@@ -29 +29 @@
 using HeuristicLab.Parameters;
 using HEAL.Attic;
+using MathNet.Numerics;
 using MathNet.Numerics.Statistics;
-
 using DoubleVector = MathNet.Numerics.LinearAlgebra.Vector<double>;
 
@@ -43 +43 @@
       Sum,
       First,
+      L1Norm,
+      L2Norm,
       NaN,
       Exception
+    }
+    public static double Aggregate(Aggregation aggregation, DoubleVector vector) {
+      switch (aggregation) {
+        case Aggregation.Mean: return Statistics.Mean(vector);
+        case Aggregation.Median: return Statistics.Median(vector);
+        case Aggregation.Sum: return vector.Sum();
+        case Aggregation.First: return vector.First();
+        case Aggregation.L1Norm: return vector.L1Norm();
+        case Aggregation.L2Norm: return vector.L2Norm();
+        case Aggregation.NaN: return double.NaN;
+        case Aggregation.Exception: throw new InvalidOperationException("Result of the tree is not a scalar.");
+        default: throw new ArgumentOutOfRangeException(nameof(aggregation), aggregation, null);
+      }
+    }
+
+    [StorableType("73DCBB45-916F-4139-8ADC-57BA610A1B66")]
+    public enum VectorLengthStrategy {
+      ExceptionIfDifferent,
+      FillShorterWithNaN,
+      FillShorterWithNeutralElement,
+      CutLonger,
+      ResampleToLonger,
+      ResampleToShorter,
+      CycleShorter
+    }
+
+    #region Implementation VectorLengthStrategy
+    public static (DoubleVector, DoubleVector) ExceptionIfDifferent(DoubleVector lhs, DoubleVector rhs) {
+      if (lhs.Count != rhs.Count)
+        throw new InvalidOperationException($"Vector Lengths incompatible ({lhs.Count} vs. {rhs.Count}");
+      return (lhs, rhs);
+    }
+
+    public static (DoubleVector, DoubleVector) FillShorter(DoubleVector lhs, DoubleVector rhs, double fillElement) {
+      var targetLength = Math.Max(lhs.Count, rhs.Count);
+
+      DoubleVector PadVector(DoubleVector v) {
+        if (v.Count == targetLength) return v;
+        var p = DoubleVector.Build.Dense(targetLength, fillElement);
+        v.CopySubVectorTo(p, 0, 0, v.Count);
+        return p;
+      }
+
+      return (PadVector(lhs), PadVector(rhs));
+    }
+
+    public static (DoubleVector, DoubleVector) CutLonger(DoubleVector lhs, DoubleVector rhs) {
+      var targetLength = Math.Min(lhs.Count, rhs.Count);
+
+      DoubleVector CutVector(DoubleVector v) {
+        if (v.Count == targetLength) return v;
+        return v.SubVector(0, targetLength);
+      }
+
+      return (CutVector(lhs), CutVector(rhs));
+    }
+
+    private static DoubleVector ResampleToLength(DoubleVector v, int targetLength) {
+      if (v.Count == targetLength) return v;
+
+      var indices = Enumerable.Range(0, v.Count).Select(x => (double)x);
+      var interpolation = Interpolate.Linear(indices, v);
+
+      var resampledIndices = Enumerable.Range(0, targetLength).Select(i => (double)i / targetLength * v.Count);
+      var interpolatedValues = resampledIndices.Select(interpolation.Interpolate);
+
+      return DoubleVector.Build.DenseOfEnumerable(interpolatedValues);
+    }
+    public static (DoubleVector, DoubleVector) ResampleToLonger(DoubleVector lhs, DoubleVector rhs) {
+      var maxLength = Math.Max(lhs.Count, rhs.Count);
+      return (ResampleToLength(lhs, maxLength), ResampleToLength(rhs, maxLength));
+    }
+    public static (DoubleVector, DoubleVector) ResampleToShorter(DoubleVector lhs, DoubleVector rhs) {
+      var minLength = Math.Min(lhs.Count, rhs.Count);
+      return (ResampleToLength(lhs, minLength), ResampleToLength(rhs, minLength));
+    }
+
+    public static (DoubleVector, DoubleVector) CycleShorter(DoubleVector lhs, DoubleVector rhs) {
+      var targetLength = Math.Max(lhs.Count, rhs.Count);
+
+      DoubleVector CycleVector(DoubleVector v) {
+        if (v.Count == targetLength) return v;
+        var cycledValues = Enumerable.Range(0, targetLength).Select(i => v[i % v.Count]);
+        return DoubleVector.Build.DenseOfEnumerable(cycledValues);
+      }
+
+      return (CycleVector(lhs), CycleVector(rhs));
+    }
+    #endregion
+
+    public static (DoubleVector lhs, DoubleVector rhs) ApplyVectorLengthStrategy(VectorLengthStrategy strategy, DoubleVector lhs, DoubleVector rhs,
+      double neutralElement = double.NaN) {
+
+      switch (strategy) {
+        case VectorLengthStrategy.ExceptionIfDifferent: return ExceptionIfDifferent(lhs, rhs);
+        case VectorLengthStrategy.FillShorterWithNaN: return FillShorter(lhs, rhs, double.NaN);
+        case VectorLengthStrategy.FillShorterWithNeutralElement: return FillShorter(lhs, rhs, neutralElement);
+        case VectorLengthStrategy.CutLonger: return CutLonger(lhs, rhs);
+        case VectorLengthStrategy.ResampleToLonger: return ResampleToLonger(lhs, rhs);
+        case VectorLengthStrategy.ResampleToShorter: return ResampleToShorter(lhs, rhs);
+        case VectorLengthStrategy.CycleShorter: return CycleShorter(lhs, rhs);
+        default: throw new ArgumentOutOfRangeException(nameof(strategy), strategy, null);
+      }
     }
 
@@ -56 +161 @@
     private const string EvaluatedSolutionsParameterName = "EvaluatedSolutions";
     private const string FinalAggregationParameterName = "FinalAggregation";
+    private const string DifferentVectorLengthStrategyParameterName = "DifferentVectorLengthStrategy";
 
     public override bool CanChangeName {
@@ -71 +177 @@
     public IFixedValueParameter<EnumValue<Aggregation>> FinalAggregationParameter {
       get { return (IFixedValueParameter<EnumValue<Aggregation>>)Parameters[FinalAggregationParameterName]; }
+    }
+    public IFixedValueParameter<EnumValue<VectorLengthStrategy>> DifferentVectorLengthStrategyParameter {
+      get { return (IFixedValueParameter<EnumValue<VectorLengthStrategy>>)Parameters[DifferentVectorLengthStrategyParameterName]; }
     }
     #endregion
@@ -83 +192 @@
       set { FinalAggregationParameter.Value.Value = value; }
     }
+    public VectorLengthStrategy DifferentVectorLengthStrategy {
+      get { return DifferentVectorLengthStrategyParameter.Value.Value; }
+      set { DifferentVectorLengthStrategyParameter.Value.Value = value; }
+    }
     #endregion
 
@@ -103 +216 @@
       Parameters.Add(new FixedValueParameter<IntValue>(EvaluatedSolutionsParameterName, "A counter for the total number of solutions the interpreter has evaluated", new IntValue(0)));
       Parameters.Add(new FixedValueParameter<EnumValue<Aggregation>>(FinalAggregationParameterName, "If root node of the expression tree results in a Vector it is aggregated according to this parameter", new EnumValue<Aggregation>(Aggregation.Mean)));
+      Parameters.Add(new FixedValueParameter<EnumValue<VectorLengthStrategy>>(DifferentVectorLengthStrategyParameterName, "", new EnumValue<VectorLengthStrategy>(VectorLengthStrategy.ExceptionIfDifferent)));
     }
 
@@ -109 +223 @@
       if (!Parameters.ContainsKey(FinalAggregationParameterName)) {
         Parameters.Add(new FixedValueParameter<EnumValue<Aggregation>>(FinalAggregationParameterName, "If root node of the expression tree results in a Vector it is aggregated according to this parameter", new EnumValue<Aggregation>(Aggregation.Mean)));
+      }
+      if (!Parameters.ContainsKey(DifferentVectorLengthStrategyParameterName)) {
+        Parameters.Add(new FixedValueParameter<EnumValue<VectorLengthStrategy>>(DifferentVectorLengthStrategyParameterName, "", new EnumValue<VectorLengthStrategy>(VectorLengthStrategy.ExceptionIfDifferent)));
       }
     }
@@ -133 +250 @@
           yield return result.Scalar;
         else if (result.IsVector) {
-          if (FinalAggregation == Aggregation.Mean) yield return result.Vector.Mean();
-          else if (FinalAggregation == Aggregation.Median) yield return Statistics.Median(result.Vector);
-          else if (FinalAggregation == Aggregation.Sum) yield return result.Vector.Sum();
-          else if (FinalAggregation == Aggregation.First) yield return result.Vector.First();
-          else if (FinalAggregation == Aggregation.Exception) throw new InvalidOperationException("Result of the tree is not a scalar.");
-          else yield return double.NaN;
+          yield return Aggregate(FinalAggregation, result.Vector);
         } else
           yield return double.NaN;
@@ -206 +318 @@
 
     private static EvaluationResult ArithmeticApply(EvaluationResult lhs, EvaluationResult rhs,
+      Func<DoubleVector, DoubleVector, (DoubleVector, DoubleVector)> lengthStrategy,
       Func<double, double, double> ssFunc = null,
       Func<double, DoubleVector, DoubleVector> svFunc = null,
       Func<DoubleVector, double, DoubleVector> vsFunc = null,
       Func<DoubleVector, DoubleVector, DoubleVector> vvFunc = null) {
+
       if (lhs.IsScalar && rhs.IsScalar && ssFunc != null) return new EvaluationResult(ssFunc(lhs.Scalar, rhs.Scalar));
       if (lhs.IsScalar && rhs.IsVector && svFunc != null) return new EvaluationResult(svFunc(lhs.Scalar, rhs.Vector));
       if (lhs.IsVector && rhs.IsScalar && vsFunc != null) return new EvaluationResult(vsFunc(lhs.Vector, rhs.Scalar));
-      if (lhs.IsVector && rhs.IsVector && vvFunc != null) return new EvaluationResult(vvFunc(lhs.Vector, rhs.Vector));
+      if (lhs.IsVector && rhs.IsVector && vvFunc != null) {
+        if (lhs.Vector.Count == rhs.Vector.Count) {
+          return new EvaluationResult(vvFunc(lhs.Vector, rhs.Vector));
+        } else {
+          var (lhsVector, rhsVector) = lengthStrategy(lhs.Vector, rhs.Vector);
+          return new EvaluationResult(vvFunc(lhsVector, rhsVector));
+        }
+      }
       return EvaluationResult.NaN;
     }
@@ -250 +371 @@
     }
     private static EvaluationResult AggregateMultipleApply(EvaluationResult lhs, EvaluationResult rhs,
+      Func<DoubleVector, DoubleVector, (DoubleVector, DoubleVector)> lengthStrategy,
       Func<double, double, double> ssFunc = null,
       Func<double, DoubleVector, double> svFunc = null,
@@ -257 +379 @@
       if (lhs.IsScalar && rhs.IsVector && svFunc != null) return new EvaluationResult(svFunc(lhs.Scalar, rhs.Vector));
       if (lhs.IsVector && rhs.IsScalar && vsFunc != null) return new EvaluationResult(vsFunc(lhs.Vector, rhs.Scalar));
-      if (lhs.IsVector && rhs.IsVector && vvFunc != null) return new EvaluationResult(vvFunc(lhs.Vector, rhs.Vector));
+      if (lhs.IsVector && rhs.IsVector && vvFunc != null) {
+        if (lhs.Vector.Count == rhs.Vector.Count) {
+          return new EvaluationResult(vvFunc(lhs.Vector, rhs.Vector));
+        } else {
+          var (lhsVector, rhsVector) = lengthStrategy(lhs.Vector, rhs.Vector);
+          return new EvaluationResult(vvFunc(lhsVector, rhsVector));
+        }
+      }
       return EvaluationResult.NaN;
     }
@@ -283 +412 @@
               var op = Evaluate(dataset, ref row, state);
               cur = ArithmeticApply(cur, op,
+                (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 0.0),
                 (s1, s2) => s1 + s2,
                 (s1, v2) => s1 + v2,
@@ -295 +425 @@
               var op = Evaluate(dataset, ref row, state);
               cur = ArithmeticApply(cur, op,
+                (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 0.0),
                 (s1, s2) => s1 - s2,
                 (s1, v2) => s1 - v2,
@@ -307 +438 @@
               var op = Evaluate(dataset, ref row, state);
               cur = ArithmeticApply(cur, op,
+                (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 1.0),
                 (s1, s2) => s1 * s2,
                 (s1, v2) => s1 * v2,
@@ -319 +451 @@
               var op = Evaluate(dataset, ref row, state);
               cur = ArithmeticApply(cur, op,
+                (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 1.0),
                 (s1, s2) => s1 / s2,
                 (s1, v2) => s1 / v2,
@@ -362 +495 @@
             var y = Evaluate(dataset, ref row, state);
             return ArithmeticApply(x, y,
+              (lhs, rhs) => lhs.Count < rhs.Count
+                ? CutLonger(lhs, rhs)
+                : ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 1.0),
               (s1, s2) => Math.Pow(s1, Math.Round(s2)),
               (s1, v2) => DoubleVector.Build.Dense(v2.Count, s1).PointwisePower(DoubleVector.Round(v2)),
@@ -383 +519 @@
             var y = Evaluate(dataset, ref row, state);
             return ArithmeticApply(x, y,
+              (lhs, rhs) => lhs.Count < rhs.Count
+                ? CutLonger(lhs, rhs)
+                : ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 1.0),
               (s1, s2) => Math.Pow(s1, 1.0 / Math.Round(s2)),
               (s1, v2) => DoubleVector.Build.Dense(v2.Count, s1).PointwisePower(1.0 / DoubleVector.Round(v2)),
@@ -410 +549 @@
             return AggregateApply(cur,
               s => s,
-              v => v.Mean());
+              v => Statistics.Mean(v));
           }
         case OpCodes.StandardDeviation: {
@@ -458 +597 @@
             var x2 = Evaluate(dataset, ref row, state);
             return AggregateMultipleApply(x1, x2,
-              //(s1, s2) => s1 - s2,
-              //(s1, v2) => Math.Sqrt((s1 - v2).PointwisePower(2).Sum()),
-              //(v1, s2) => Math.Sqrt((v1 - s2).PointwisePower(2).Sum()),
-              vvFunc: (v1, v2) => v1.Count == v2.Count ? Math.Sqrt((v1 - v2).PointwisePower(2).Sum()) : double.NaN);
+              (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 0.0),
+              (s1, s2) => s1 - s2,
+              (s1, v2) => Math.Sqrt((s1 - v2).PointwisePower(2).Sum()),
+              (v1, s2) => Math.Sqrt((v1 - s2).PointwisePower(2).Sum()),
+              (v1, v2) => Math.Sqrt((v1 - v2).PointwisePower(2).Sum()));
           }
         case OpCodes.Covariance: {
@@ -467 +607 @@
             var x2 = Evaluate(dataset, ref row, state);
             return AggregateMultipleApply(x1, x2,
-              //(s1, s2) => 0,
-              //(s1, v2) => 0,
-              //(v1, s2) => 0,
-              vvFunc: (v1, v2) => v1.Count == v2.Count ? Statistics.PopulationCovariance(v1, v2) : double.NaN);
+              (lhs, rhs) => ApplyVectorLengthStrategy(DifferentVectorLengthStrategy, lhs, rhs, 0.0),
+              (s1, s2) => 0,
+              (s1, v2) => 0,
+              (v1, s2) => 0,
+              (v1, v2) => Statistics.PopulationCovariance(v1, v2));
           }
         case OpCodes.Variable: {

branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VectorData/AzzaliBenchmark3.cs

@@ -33 +33 @@
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
   public class AzzaliBenchmark3 : ArtificialRegressionDataDescriptor {
-    public override string Name { get { return "Azzali Benchmark3 B3 = CumMin[3,3] * (X2 / X3) + X4"; } }
+    public override string Name { get { return "Azzali Benchmark3 B3 = CumMin[3,3](X1) * (X2 / X3) + X4"; } }
     public override string Description { get { return "I. Azzali, L. Vanneschi, S. Silva, I. Bakurov, and M. Giacobini, “A Vectorial Approach to Genetic Programming,” EuroGP, pp. 213–227, 2019."; } }