Changeset 16518 for branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis

Timestamp:

01/08/19 14:59:31 (6 years ago)

Author:

pfleck

Message:

#2972 merged trunk into branch

Location:

branches/2972_PDPRowSelect

Files:

• . (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4 (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegressionModel.cs (modified) (8 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs (modified) (6 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs (modified) (9 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs (modified) (6 diffs)

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs

@@ -41 +41 @@
   [StorableClass]
   public sealed class LinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
-    private const string LinearRegressionModelResultName = "Linear regression solution";
+    private const string SolutionResultName = "Linear regression solution";
+    private const string ConfidenceSolutionResultName = "Solution with prediction intervals";
 
     [StorableConstructor]
@@ -62 +63 @@
     protected override void Run(CancellationToken cancellationToken) {
       double rmsError, cvRmsError;
+      // produce both solutions, to allow symbolic manipulation of LR solutions as well
+      // as the calculation of prediction intervals.
+      // There is no clean way to implement the new model class for LR as a symbolic model.
       var solution = CreateSolution(Problem.ProblemData, out rmsError, out cvRmsError);
-      Results.Add(new Result(LinearRegressionModelResultName, "The linear regression solution.", solution));
+#pragma warning disable 168, 3021
+      var symbolicSolution = CreateLinearRegressionSolution(Problem.ProblemData, out rmsError, out cvRmsError);
+#pragma warning restore 168, 3021
+      Results.Add(new Result(SolutionResultName, "The linear regression solution.", symbolicSolution));
+      Results.Add(new Result(ConfidenceSolutionResultName, "Linear regression solution with parameter covariance matrix " +
+                                                           "and calculation of prediction intervals", solution));
       Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the linear regression solution on the training set.", new DoubleValue(rmsError)));
       Results.Add(new Result("Estimated root mean square error (cross-validation)", "The estimated root of the mean of squared errors of the linear regression solution via cross validation.", new DoubleValue(cvRmsError)));
@@ -88 +97 @@
       double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
       alglib.lrunpack(lm, out coefficients, out nFeatures);
-      
-      int nFactorCoeff = factorVariables.Sum(kvp=>kvp.Value.Count());
+
+      int nFactorCoeff = factorVariables.Sum(kvp => kvp.Value.Count());
       int nVarCoeff = doubleVariables.Count();
       var tree = LinearModelToTreeConverter.CreateTree(factorVariables, coefficients.Take(nFactorCoeff).ToArray(),
@@ -132 +141 @@
     }
 
-    private static void PrepareData(IRegressionProblemData problemData, 
-      out double[,] inputMatrix, 
-      out IEnumerable<string> doubleVariables, 
+    private static void PrepareData(IRegressionProblemData problemData,
+      out double[,] inputMatrix,
+      out IEnumerable<string> doubleVariables,
       out IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables) {
       var dataset = problemData.Dataset;

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegressionModel.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Drawing;
 using System.Linq;
 using HeuristicLab.Common;
@@ -35 +36 @@
   [Item("Linear Regression Model", "Represents a linear regression model.")]
   public sealed class LinearRegressionModel : RegressionModel, IConfidenceRegressionModel {
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Function; }
+    }
 
     [Storable]
@@ -49 +53 @@
       get; private set;
     }
-    
+
     public override IEnumerable<string> VariablesUsedForPrediction {
-      get { return allowedInputVariables.Union(factorVariables.Select(f => f.Key)); }
+      get { return doubleVariables.Union(factorVariables.Select(f => f.Key)); }
     }
 
     [Storable]
-    private string[] allowedInputVariables;
+    private string[] doubleVariables;
     [Storable]
     private List<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+
+    /// <summary>
+    /// Enumerable of variable names used by the model including one-hot-encoded of factor variables.
+    /// </summary>
+    public IEnumerable<string> ParameterNames {
+      get {
+        return factorVariables.SelectMany(kvp => kvp.Value.Select(factorVal => $"{kvp.Key}={factorVal}"))
+          .Concat(doubleVariables)
+          .Concat(new[] { "<const>" });
+      }
+    }
 
     [StorableConstructor]
@@ -69 +84 @@
       this.NoiseSigma = original.NoiseSigma;
 
-      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      doubleVariables = (string[])original.doubleVariables.Clone();
       this.factorVariables = original.factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
     }
@@ -78 +93 @@
       this.W = new double[w.Length];
       Array.Copy(w, W, w.Length);
-      this.C = new double[covariance.GetLength(0),covariance.GetLength(1)];
+      this.C = new double[covariance.GetLength(0), covariance.GetLength(1)];
       Array.Copy(covariance, C, covariance.Length);
       this.NoiseSigma = noiseSigma;
-      var stringInputVariables = factorVariables.Select(f => f.Key).Distinct();
-      this.allowedInputVariables = doubleInputVariables.ToArray();
+      this.doubleVariables = doubleInputVariables.ToArray();
+      // clone
       this.factorVariables = factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
     }
@@ -95 +110 @@
 
     public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(doubleVariables, rows);
       double[,] factorData = dataset.ToArray(factorVariables, rows);
 
@@ -114 +129 @@
 
     public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(doubleVariables, rows);
       double[,] factorData = dataset.ToArray(factorVariables, rows);
 
@@ -123 +138 @@
 
       double[] d = new double[C.GetLength(0)];
-      
+
       for (int row = 0; row < n; row++) {
         for (int column = 0; column < columns; column++) {
-          d[column] = inputData[row,column];
+          d[column] = inputData[row, column];
         }
         d[columns] = 1;
 
         double var = 0.0;
-        for(int i=0;i<d.Length;i++) {
-          for(int j = 0;j<d.Length;j++) {
+        for (int i = 0; i < d.Length; i++) {
+          for (int j = 0; j < d.Length; j++) {
             var += d[i] * C[i, j] * d[j];
           }
         }
-        yield return var + NoiseSigma*NoiseSigma;
+        yield return var + NoiseSigma * NoiseSigma;
       }
     }
-
 
     public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs

@@ -65 +65 @@
 
       var ds = ReduceDataset(dataset, rows);
-      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues: classValues);
+      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, false, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues: classValues);
     }
 

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs

@@ -1 @@
-#region License Information
+#region License Information
 /* HeuristicLab
  * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
@@ -42 +42 @@
     private const string NearestNeighbourClassificationModelResultName = "Nearest neighbour classification solution";
     private const string WeightsParameterName = "Weights";
-
+    private const string SelfMatchParameterName = "SelfMatch";
 
     #region parameter properties
     public IFixedValueParameter<IntValue> KParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SelfMatchParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
     }
     public IValueParameter<DoubleArray> WeightsParameter {
@@ -53 +56 @@
     #endregion
     #region properties
+    public bool SelfMatch {
+      get { return SelfMatchParameter.Value.Value; }
+      set { SelfMatchParameter.Value.Value = value; }
+    }
     public int K {
       get { return KParameter.Value.Value; }
@@ -73 +80 @@
     public NearestNeighbourClassification()
       : base() {
+      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
       Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
@@ -83 +91 @@
       if (!Parameters.ContainsKey(WeightsParameterName)) {
         Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      }
+      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       }
       #endregion
@@ -95 +106 @@
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();
-      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K, weights);
+      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K, SelfMatch, weights);
       Results.Add(new Result(NearestNeighbourClassificationModelResultName, "The nearest neighbour classification solution.", solution));
     }
 
-    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k, double[] weights = null) {
+    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
       var problemDataClone = (IClassificationProblemData)problemData.Clone();
-      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k, weights), problemDataClone);
+      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k, selfMatch, weights), problemDataClone);
     }
 
-    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k, double[] weights = null) {
+    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
       return new NearestNeighbourModel(problemData.Dataset,
         problemData.TrainingIndices,
         k,
+        selfMatch,
         problemData.TargetVariable,
         problemData.AllowedInputVariables,

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs

@@ -1 @@
-#region License Information
+#region License Information
 /* HeuristicLab
  * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
@@ -37 +37 @@
 
     private readonly object kdTreeLockObject = new object();
+
     private alglib.nearestneighbor.kdtree kdTree;
     public alglib.nearestneighbor.kdtree KDTree {
@@ -49 +50 @@
     }
 
-
     public override IEnumerable<string> VariablesUsedForPrediction {
       get { return allowedInputVariables; }
@@ -60 +60 @@
     [Storable]
     private int k;
+    [Storable(DefaultValue = false)]
+    private bool selfMatch;
     [Storable(DefaultValue = null)]
     private double[] weights; // not set for old versions loaded from disk
@@ -97 +99 @@
       kdTree.x = (double[])original.kdTree.x.Clone();
       kdTree.xy = (double[,])original.kdTree.xy.Clone();
-
+      selfMatch = original.selfMatch;
       k = original.k;
       isCompatibilityLoaded = original.IsCompatibilityLoaded;
@@ -110 +112 @@
         this.classValues = (double[])original.classValues.Clone();
     }
-    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> weights = null, double[] classValues = null)
+    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, bool selfMatch, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> weights = null, double[] classValues = null)
       : base(targetVariable) {
       Name = ItemName;
       Description = ItemDescription;
+      this.selfMatch = selfMatch;
       this.k = k;
       this.allowedInputVariables = allowedInputVariables.ToArray();
@@ -132 +135 @@
             .Select(name => {
               var pop = dataset.GetDoubleValues(name, rows).StandardDeviationPop();
-              return  pop.IsAlmost(0) ? 1.0 : 1.0/pop;
+              return pop.IsAlmost(0) ? 1.0 : 1.0 / pop;
             })
             .Concat(new double[] { 1.0 }) // no scaling for target variable
@@ -201 +204 @@
         int numNeighbours;
         lock (kdTreeLockObject) { // gkronber: the following calls change the kdTree data structure
-          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, false);
+          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
           alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
           alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
         }
-
+        if (selfMatch) {
+          // weights for neighbours are 1/d.
+          // override distances (=0) of exact matches using 1% of the distance of the next closest non-self-match neighbour -> selfmatches weight 100x more than the next closest neighbor.
+          // if all k neighbours are selfmatches then they all have weight 0.01.
+          double minDist = dists[0] + 1;
+          for (int i = 0; i < numNeighbours; i++) {
+            if ((minDist > dists[i]) && (dists[i] != 0)) {
+              minDist = dists[i];
+            }
+          }
+          minDist /= 100.0;
+          for (int i = 0; i < numNeighbours; i++) {
+            if (dists[i] == 0) {
+              dists[i] = minDist;
+            }
+          }
+        }
         double distanceWeightedValue = 0.0;
         double distsSum = 0.0;
@@ -238 +257 @@
         lock (kdTreeLockObject) {
           // gkronber: the following calls change the kdTree data structure
-          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, false);
+          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
           alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
           alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);

branches/2972_PDPRowSelect/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs

@@ -41 +41 @@
     private const string NearestNeighbourRegressionModelResultName = "Nearest neighbour regression solution";
     private const string WeightsParameterName = "Weights";
+    private const string SelfMatchParameterName = "SelfMatch";
 
     #region parameter properties
@@ -46 +47 @@
       get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
     }
-
+    public IFixedValueParameter<BoolValue> SelfMatchParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
+    }
     public IValueParameter<DoubleArray> WeightsParameter {
       get { return (IValueParameter<DoubleArray>)Parameters[WeightsParameterName]; }
@@ -59 +62 @@
       }
     }
-
+    public bool SelfMatch {
+      get { return SelfMatchParameter.Value.Value; }
+      set { SelfMatchParameter.Value.Value = value; }
+    }
     public DoubleArray Weights {
       get { return WeightsParameter.Value; }
@@ -75 +81 @@
       Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
       Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       Problem = new RegressionProblem();
     }
@@ -84 +91 @@
       if (!Parameters.ContainsKey(WeightsParameterName)) {
         Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
+      }
+      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       }
       #endregion
@@ -96 +106 @@
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();
-      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K, weights);
+      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K, SelfMatch, weights);
       Results.Add(new Result(NearestNeighbourRegressionModelResultName, "The nearest neighbour regression solution.", solution));
     }
 
-    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k, double[] weights = null) {
+    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
       var clonedProblemData = (IRegressionProblemData)problemData.Clone();
-      return new NearestNeighbourRegressionSolution(Train(problemData, k, weights), clonedProblemData);
+      return new NearestNeighbourRegressionSolution(Train(problemData, k, selfMatch, weights), clonedProblemData);
     }
 
-    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k, double[] weights = null) {
+    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
       return new NearestNeighbourModel(problemData.Dataset,
         problemData.TrainingIndices,
         k,
+        selfMatch,
         problemData.TargetVariable,
         problemData.AllowedInputVariables,