Changeset 17970 for branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4

Timestamp:

04/30/21 12:18:19 (3 years ago)

Author:

gkronber

Message:

#3106 merged r17856:17969 from trunk to branch

Location:

branches/3106_AnalyticContinuedFractionsRegression

Files:

• . (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4 (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs (modified) (7 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs (modified) (3 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs (modified) (5 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs (modified) (6 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModelAlglib_3_7.cs (copied) (copied from trunk/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModelAlglib_3_7.cs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs (modified) (5 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs (modified) (7 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModelAlglib_3_7.cs (copied) (copied from trunk/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModelAlglib_3_7.cs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs (modified) (9 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelAlglib_3_7.cs (copied) (copied from trunk/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelAlglib_3_7.cs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelFull.cs (modified) (7 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelSurrogate.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs (modified) (8 diffs)

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs

@@ -39 +39 @@
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 600)]
   public sealed class GeneralizedAdditiveModelAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+
     #region ParameterNames
 
@@ -46 +47 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string CreateSolutionParameterName = "CreateSolution";
+
     #endregion
 
@@ -141 +143 @@
       var problemData = Problem.ProblemData;
       var ds = problemData.Dataset;
-      var trainRows = problemData.TrainingIndices;
-      var testRows = problemData.TestIndices;
+      var trainRows = problemData.TrainingIndices.ToArray();
+      var testRows = problemData.TestIndices.ToArray();
       var avgY = problemData.TargetVariableTrainingValues.Average();
       var inputVars = problemData.AllowedInputVariables.ToArray();
@@ -178 +180 @@
       double[] resTest = problemData.TargetVariableTestValues.Select(yi => yi - avgY).ToArray();
 
-      curRMSE.Value = res.StandardDeviation();
-      curRMSETest.Value = resTest.StandardDeviation();
-      rmseRow.Values.Add(res.StandardDeviation()); 
-      rmseRowTest.Values.Add(resTest.StandardDeviation());
+      curRMSE.Value = RMSE(res);
+      curRMSETest.Value = RMSE(resTest);
+      rmseRow.Values.Add(curRMSE.Value);
+      rmseRowTest.Values.Add(curRMSETest.Value);
 
 
@@ -197 +199 @@
           AddInPlace(resTest, f[inputIdx].GetEstimatedValues(ds, testRows));
 
-          rssTable[inputIdx, 0] = res.Variance();
+          rssTable[inputIdx, 0] = MSE(res);
           f[inputIdx] = RegressSpline(problemData, inputVar, res, lambda);
 
@@ -204 +206 @@
         }
 
-        curRMSE.Value = res.StandardDeviation(); 
-        curRMSETest.Value = resTest.StandardDeviation();
+        curRMSE.Value = RMSE(res);
+        curRMSETest.Value = RMSE(resTest);
         rmseRow.Values.Add(curRMSE.Value);
         rmseRowTest.Values.Add(curRMSETest.Value);
@@ -215 +217 @@
         var model = new RegressionEnsembleModel(f.Concat(new[] { new ConstantModel(avgY, problemData.TargetVariable) }));
         model.AverageModelEstimates = false;
-        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
+        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());        
         Results.Add(new Result("Ensemble solution", solution));
       }
+    }
+
+    public static double MSE(IEnumerable<double> residuals) {
+      var mse  = residuals.Select(r => r * r).Average();
+      return mse;
+    }
+
+    public static double RMSE(IEnumerable<double> residuals) {
+      var mse = MSE(residuals);
+      var rmse = Math.Sqrt(mse);
+      return rmse;
     }
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs

@@ -26 +26 @@
 using HeuristicLab.Core;
 using HeuristicLab.Problems.DataAnalysis;
+using System;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
@@ -35 +36 @@
     private alglib.spline1d.spline1dinterpolant interpolant;
 
-    [Storable]
-    private readonly string[] variablesUsedForPrediction;
-    public override IEnumerable<string> VariablesUsedForPrediction {
-      get {
-        return variablesUsedForPrediction;
+    [Storable(OldName = "variablesUsedForPrediction")]
+    private string[] StorableVariablesUsedForPrediction {
+      set {
+        if (value.Length > 1) throw new ArgumentException("A one-dimensional spline model supports only one input variable.");
+        inputVariable = value[0];
       }
     }
+
+    [Storable]
+    private string inputVariable;
+    public override IEnumerable<string> VariablesUsedForPrediction => new[] { inputVariable };
 
     [StorableConstructor]
@@ -49 +54 @@
 
     private Spline1dModel(Spline1dModel orig, Cloner cloner) : base(orig, cloner) {
-      this.variablesUsedForPrediction = orig.VariablesUsedForPrediction.ToArray();
-      this.interpolant = (alglib.spline1d.spline1dinterpolant)orig.interpolant.make_copy();
+      this.inputVariable = orig.inputVariable;
+      if(orig.interpolant != null) this.interpolant = (alglib.spline1d.spline1dinterpolant)orig.interpolant.make_copy();
     }
     public Spline1dModel(alglib.spline1d.spline1dinterpolant interpolant, string targetVar, string inputVar)
-      : base(targetVar, "Spline model (1d)") {
-      this.interpolant = (alglib.spline1d.spline1dinterpolant)interpolant.make_copy();      
-      this.variablesUsedForPrediction = new string[] { inputVar };
+      : base(targetVar, $"Spline model ({inputVar})") {
+      this.interpolant = (alglib.spline1d.spline1dinterpolant)interpolant.make_copy();
+      this.inputVariable = inputVar;
     }
 
 
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new Spline1dModel(this, cloner);
+    public override IDeepCloneable Clone(Cloner cloner) => new Spline1dModel(this, cloner);
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      var solution =  new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+      solution.Name = $"Regression Spline ({inputVariable})";
+
+      return solution;
     }
 
-    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
-    }
-
-    public double GetEstimatedValue(double x) {
-      return alglib.spline1d.spline1dcalc(interpolant, x);
-    }
+    public double GetEstimatedValue(double x) => alglib.spline1d.spline1dcalc(interpolant, x, null);
 
     public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      var x = dataset.GetDoubleValues(VariablesUsedForPrediction.First(), rows).ToArray();
-      foreach (var xi in x) {
-        yield return GetEstimatedValue(xi);
-      }
+      return dataset.GetDoubleValues(inputVariable, rows).Select(GetEstimatedValue);
     }
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

@@ -321 +321 @@
 
       // cholesky decomposition
-      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false, null);
       if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
       return l;
@@ -412 +412 @@
 
         // for stddev 
-        alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
+        alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, sWKs, 0, 0, null);
 
         for (int i = 0; i < newN; i++) {

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -107 +107 @@
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ALGLIB-3.17.0, Version=3.17.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\ALGLIB-3.17.0.dll</HintPath>
+      <Aliases>global</Aliases>
+      <Private>False</Private>
+    </Reference>
     <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\bin\ALGLIB-3.7.0.dll</HintPath>
+      <Aliases>alglib_3_7</Aliases>
       <Private>False</Private>
     </Reference>
@@ -294 +302 @@
     <Compile Include="NearestNeighbour\NearestNeighbourClassificationSolution.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourModel.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourModelAlglib_3_7.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourRegression.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourRegressionSolution.cs" />
@@ -304 +313 @@
     <Compile Include="NeuralNetwork\NeuralNetworkClassificationSolution.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkModelAlglib_3_7.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkRegression.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkRegressionSolution.cs" />
@@ -313 +323 @@
     <Compile Include="RandomForest\RandomForestClassification.cs" />
     <Compile Include="RandomForest\RandomForestModel.cs" />
+    <Compile Include="RandomForest\RandomForestModelAlglib_3_7.cs" />
     <Compile Include="RandomForest\RandomForestModelFull.cs" />
     <Compile Include="RandomForest\RandomForestRegression.cs" />

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/KernelRidgeRegression/KernelRidgeRegressionModel.cs

@@ -108 +108 @@
         }
         // cholesky decomposition
-        var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+        var res = alglib.trfac.spdmatrixcholesky(ref l, n, false, null);
         if (res == false) { //try lua decomposition if cholesky faild
           int[] pivots;

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs

@@ -65 +65 @@
 
       var ds = ReduceDataset(dataset, rows);
-      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, false, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues: classValues);
+      // the new implementation of kNN uses selfmatch=true by default
+      nnModel = new NearestNeighbourModelAlglib_3_7(ds, Enumerable.Range(0, ds.Rows), k, false, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues: classValues);
     }
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs

@@ -42 +42 @@
     private const string NearestNeighbourClassificationModelResultName = "Nearest neighbour classification solution";
     private const string WeightsParameterName = "Weights";
-    private const string SelfMatchParameterName = "SelfMatch";
 
     #region parameter properties
     public IFixedValueParameter<IntValue> KParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
-    }
-    public IFixedValueParameter<BoolValue> SelfMatchParameter {
-      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
     }
     public IValueParameter<DoubleArray> WeightsParameter {
@@ -56 +52 @@
     #endregion
     #region properties
-    public bool SelfMatch {
-      get { return SelfMatchParameter.Value.Value; }
-      set { SelfMatchParameter.Value.Value = value; }
-    }
     public int K {
       get { return KParameter.Value.Value; }
@@ -80 +72 @@
     public NearestNeighbourClassification()
       : base() {
-      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
       Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
@@ -91 +82 @@
       if (!Parameters.ContainsKey(WeightsParameterName)) {
         Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
-      }
-      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
-        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       }
       #endregion
@@ -106 +94 @@
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();
-      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K, SelfMatch, weights);
+      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K, weights);
       Results.Add(new Result(NearestNeighbourClassificationModelResultName, "The nearest neighbour classification solution.", solution));
     }
 
-    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k, double[] weights = null) {
       var problemDataClone = (IClassificationProblemData)problemData.Clone();
-      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k, selfMatch, weights), problemDataClone);
+      return new NearestNeighbourClassificationSolution(Train(problemDataClone, k, weights), problemDataClone);
     }
 
-    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k, double[] weights = null) {
       return new NearestNeighbourModel(problemData.Dataset,
         problemData.TrainingIndices,
         k,
-        selfMatch,
         problemData.TargetVariable,
         problemData.AllowedInputVariables,

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs

@@ -32 +32 @@
   /// Represents a nearest neighbour model for regression and classification
   /// </summary>
-  [StorableType("A76C0823-3077-4ACE-8A40-E9B717C7DB60")]
+  [StorableType("04A07DF6-6EB5-4D29-B7AE-5BE204CAF6BC")]
   [Item("NearestNeighbourModel", "Represents a nearest neighbour model for regression and classification.")]
   public sealed class NearestNeighbourModel : ClassificationModel, INearestNeighbourModel {
 
-    private readonly object kdTreeLockObject = new object();
-
-    private alglib.nearestneighbor.kdtree kdTree;
-    public alglib.nearestneighbor.kdtree KDTree {
-      get { return kdTree; }
-      set {
-        if (value != kdTree) {
-          if (value == null) throw new ArgumentNullException();
-          kdTree = value;
-          OnChanged(EventArgs.Empty);
-        }
-      }
+    private alglib.knnmodel model;
+    [Storable]
+    private string SerializedModel {
+      get { alglib.knnserialize(model, out var ser); return ser; }
+      set { if (value != null) alglib.knnunserialize(value, out model); }
     }
 
@@ -60 +53 @@
     [Storable]
     private int k;
-    [Storable(DefaultValue = false)]
-    private bool selfMatch;
-    [Storable(DefaultValue = null)]
-    private double[] weights; // not set for old versions loaded from disk
-    [Storable(DefaultValue = null)]
-    private double[] offsets; // not set for old versions loaded from disk
+    [Storable]
+    private double[] weights;
+    [Storable]
+    private double[] offsets;
 
     [StorableConstructor]
-    private NearestNeighbourModel(StorableConstructorFlag _) : base(_) {
-      kdTree = new alglib.nearestneighbor.kdtree();
-    }
+    private NearestNeighbourModel(StorableConstructorFlag _) : base(_) { }
     private NearestNeighbourModel(NearestNeighbourModel original, Cloner cloner)
       : base(original, cloner) {
-      kdTree = new alglib.nearestneighbor.kdtree();
-      kdTree.approxf = original.kdTree.approxf;
-      kdTree.boxmax = (double[])original.kdTree.boxmax.Clone();
-      kdTree.boxmin = (double[])original.kdTree.boxmin.Clone();
-      kdTree.buf = (double[])original.kdTree.buf.Clone();
-      kdTree.curboxmax = (double[])original.kdTree.curboxmax.Clone();
-      kdTree.curboxmin = (double[])original.kdTree.curboxmin.Clone();
-      kdTree.curdist = original.kdTree.curdist;
-      kdTree.debugcounter = original.kdTree.debugcounter;
-      kdTree.idx = (int[])original.kdTree.idx.Clone();
-      kdTree.kcur = original.kdTree.kcur;
-      kdTree.kneeded = original.kdTree.kneeded;
-      kdTree.n = original.kdTree.n;
-      kdTree.nodes = (int[])original.kdTree.nodes.Clone();
-      kdTree.normtype = original.kdTree.normtype;
-      kdTree.nx = original.kdTree.nx;
-      kdTree.ny = original.kdTree.ny;
-      kdTree.r = (double[])original.kdTree.r.Clone();
-      kdTree.rneeded = original.kdTree.rneeded;
-      kdTree.selfmatch = original.kdTree.selfmatch;
-      kdTree.splits = (double[])original.kdTree.splits.Clone();
-      kdTree.tags = (int[])original.kdTree.tags.Clone();
-      kdTree.x = (double[])original.kdTree.x.Clone();
-      kdTree.xy = (double[,])original.kdTree.xy.Clone();
-      selfMatch = original.selfMatch;
+      if (original.model != null)
+        model = (alglib.knnmodel)original.model.make_copy();
       k = original.k;
-      isCompatibilityLoaded = original.IsCompatibilityLoaded;
-      if (!IsCompatibilityLoaded) {
-        weights = new double[original.weights.Length];
-        Array.Copy(original.weights, weights, weights.Length);
-        offsets = new double[original.offsets.Length];
-        Array.Copy(original.offsets, this.offsets, this.offsets.Length);
-      }
+      weights = new double[original.weights.Length];
+      Array.Copy(original.weights, weights, weights.Length);
+      offsets = new double[original.offsets.Length];
+      Array.Copy(original.offsets, this.offsets, this.offsets.Length);
+
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       if (original.classValues != null)
         this.classValues = (double[])original.classValues.Clone();
     }
-    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, bool selfMatch, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> weights = null, double[] classValues = null)
+    public NearestNeighbourModel(IDataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> weights = null, double[] classValues = null)
       : base(targetVariable) {
       Name = ItemName;
       Description = ItemDescription;
-      this.selfMatch = selfMatch;
       this.k = k;
       this.allowedInputVariables = allowedInputVariables.ToArray();
       double[,] inputMatrix;
-      if (IsCompatibilityLoaded) {
-        // no scaling
-        inputMatrix = dataset.ToArray(
-          this.allowedInputVariables.Concat(new string[] { targetVariable }),
-          rows);
+      this.offsets = this.allowedInputVariables
+        .Select(name => dataset.GetDoubleValues(name, rows).Average() * -1)
+        .Concat(new double[] { 0 }) // no offset for target variable
+        .ToArray();
+      if (weights == null) {
+        // automatic determination of weights (all features should have variance = 1)
+        this.weights = this.allowedInputVariables
+          .Select(name => {
+            var pop = dataset.GetDoubleValues(name, rows).StandardDeviationPop();
+            return pop.IsAlmost(0) ? 1.0 : 1.0 / pop;
+          })
+          .Concat(new double[] { 1.0 }) // no scaling for target variable
+          .ToArray();
       } else {
-        this.offsets = this.allowedInputVariables
-          .Select(name => dataset.GetDoubleValues(name, rows).Average() * -1)
-          .Concat(new double[] { 0 }) // no offset for target variable
-          .ToArray();
-        if (weights == null) {
-          // automatic determination of weights (all features should have variance = 1)
-          this.weights = this.allowedInputVariables
-            .Select(name => {
-              var pop = dataset.GetDoubleValues(name, rows).StandardDeviationPop();
-              return pop.IsAlmost(0) ? 1.0 : 1.0 / pop;
-            })
-            .Concat(new double[] { 1.0 }) // no scaling for target variable
-            .ToArray();
-        } else {
-          // user specified weights (+ 1 for target)
-          this.weights = weights.Concat(new double[] { 1.0 }).ToArray();
-          if (this.weights.Length - 1 != this.allowedInputVariables.Length)
-            throw new ArgumentException("The number of elements in the weight vector must match the number of input variables");
-        }
-        inputMatrix = CreateScaledData(dataset, this.allowedInputVariables.Concat(new string[] { targetVariable }), rows, this.offsets, this.weights);
-      }
+        // user specified weights (+ 1 for target)
+        this.weights = weights.Concat(new double[] { 1.0 }).ToArray();
+        if (this.weights.Length - 1 != this.allowedInputVariables.Length)
+          throw new ArgumentException("The number of elements in the weight vector must match the number of input variables");
+      }
+      inputMatrix = CreateScaledData(dataset, this.allowedInputVariables.Concat(new string[] { targetVariable }), rows, this.offsets, this.weights);
 
       if (inputMatrix.ContainsNanOrInfinity())
         throw new NotSupportedException(
           "Nearest neighbour model does not support NaN or infinity values in the input dataset.");
-
-      this.kdTree = new alglib.nearestneighbor.kdtree();
 
       var nRows = inputMatrix.GetLength(0);
@@ -167 +121 @@
         }
       }
-      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdTree);
+
+      alglib.knnbuildercreate(out var knnbuilder);
+      if (classValues == null) {
+        alglib.knnbuildersetdatasetreg(knnbuilder, inputMatrix, nRows, nFeatures, nout: 1);
+      } else {
+        alglib.knnbuildersetdatasetcls(knnbuilder, inputMatrix, nRows, nFeatures, classValues.Length);
+      }
+      alglib.knnbuilderbuildknnmodel(knnbuilder, k, 0.0, out model, out var report); // eps=0 (exact k-nn search is performed)
+
     }
 
@@ -184 +146 @@
     public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
       double[,] inputData;
-      if (IsCompatibilityLoaded) {
-        inputData = dataset.ToArray(allowedInputVariables, rows);
-      } else {
-        inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
-      }
+      inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
 
       int n = inputData.GetLength(0);
       int columns = inputData.GetLength(1);
       double[] x = new double[columns];
-      double[] dists = new double[k];
-      double[,] neighbours = new double[k, columns + 1];
-
+
+      alglib.knncreatebuffer(model, out var buf);
+      var y = new double[1];
       for (int row = 0; row < n; row++) {
         for (int column = 0; column < columns; column++) {
           x[column] = inputData[row, column];
         }
-        int numNeighbours;
-        lock (kdTreeLockObject) { // gkronber: the following calls change the kdTree data structure
-          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
-          alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
-          alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
-        }
-        if (selfMatch) {
-          // weights for neighbours are 1/d.
-          // override distances (=0) of exact matches using 1% of the distance of the next closest non-self-match neighbour -> selfmatches weight 100x more than the next closest neighbor.
-          // if all k neighbours are selfmatches then they all have weight 0.01.
-          double minDist = dists[0] + 1;
-          for (int i = 0; i < numNeighbours; i++) {
-            if ((minDist > dists[i]) && (dists[i] != 0)) {
-              minDist = dists[i];
-            }
-          }
-          minDist /= 100.0;
-          for (int i = 0; i < numNeighbours; i++) {
-            if (dists[i] == 0) {
-              dists[i] = minDist;
-            }
-          }
-        }
-        double distanceWeightedValue = 0.0;
-        double distsSum = 0.0;
-        for (int i = 0; i < numNeighbours; i++) {
-          distanceWeightedValue += neighbours[i, columns] / dists[i];
-          distsSum += 1.0 / dists[i];
-        }
-        yield return distanceWeightedValue / distsSum;
+        alglib.knntsprocess(model, buf, x, ref y); // thread-safe process
+        yield return y[0];
       }
     }
@@ -236 +166 @@
       if (classValues == null) throw new InvalidOperationException("No class values are defined.");
       double[,] inputData;
-      if (IsCompatibilityLoaded) {
-        inputData = dataset.ToArray(allowedInputVariables, rows);
-      } else {
-        inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
-      }
+      inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
+
       int n = inputData.GetLength(0);
       int columns = inputData.GetLength(1);
       double[] x = new double[columns];
-      int[] y = new int[classValues.Length];
-      double[] dists = new double[k];
-      double[,] neighbours = new double[k, columns + 1];
-
+
+      alglib.knncreatebuffer(model, out var buf);
+      var y = new double[classValues.Length];
       for (int row = 0; row < n; row++) {
         for (int column = 0; column < columns; column++) {
           x[column] = inputData[row, column];
         }
-        int numNeighbours;
-        lock (kdTreeLockObject) {
-          // gkronber: the following calls change the kdTree data structure
-          numNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, selfMatch);
-          alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
-          alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
-        }
-        Array.Clear(y, 0, y.Length);
-        for (int i = 0; i < numNeighbours; i++) {
-          int classValue = (int)Math.Round(neighbours[i, columns]);
-          y[classValue]++;
-        }
-
-        // find class for with the largest probability value
-        int maxProbClassIndex = 0;
-        double maxProb = y[0];
-        for (int i = 1; i < y.Length; i++) {
-          if (maxProb < y[i]) {
-            maxProb = y[i];
-            maxProbClassIndex = i;
-          }
-        }
-        yield return classValues[maxProbClassIndex];
+        alglib.knntsprocess(model, buf, x, ref y); // thread-safe process
+        // find most probably class
+        var maxC = 0;
+        for (int i = 1; i < y.Length; i++)
+          if (maxC < y[i]) maxC = i;
+        yield return classValues[maxC];
       }
     }
@@ -303 +212 @@
       return new NearestNeighbourClassificationSolution(this, new ClassificationProblemData(problemData));
     }
-
-    #region events
-    public event EventHandler Changed;
-    private void OnChanged(EventArgs e) {
-      var handlers = Changed;
-      if (handlers != null)
-        handlers(this, e);
-    }
-    #endregion
-
-
-    // BackwardsCompatibility3.3
-    #region Backwards compatible code, remove with 3.4
-
-    private bool isCompatibilityLoaded = false; // new kNN models have the value false, kNN models loaded from disc have the value true
-    [Storable(DefaultValue = true)]
-    public bool IsCompatibilityLoaded {
-      get { return isCompatibilityLoaded; }
-      set { isCompatibilityLoaded = value; }
-    }
-    #endregion
-    #region persistence
-    [Storable]
-    public double KDTreeApproxF {
-      get { return kdTree.approxf; }
-      set { kdTree.approxf = value; }
-    }
-    [Storable]
-    public double[] KDTreeBoxMax {
-      get { return kdTree.boxmax; }
-      set { kdTree.boxmax = value; }
-    }
-    [Storable]
-    public double[] KDTreeBoxMin {
-      get { return kdTree.boxmin; }
-      set { kdTree.boxmin = value; }
-    }
-    [Storable]
-    public double[] KDTreeBuf {
-      get { return kdTree.buf; }
-      set { kdTree.buf = value; }
-    }
-    [Storable]
-    public double[] KDTreeCurBoxMax {
-      get { return kdTree.curboxmax; }
-      set { kdTree.curboxmax = value; }
-    }
-    [Storable]
-    public double[] KDTreeCurBoxMin {
-      get { return kdTree.curboxmin; }
-      set { kdTree.curboxmin = value; }
-    }
-    [Storable]
-    public double KDTreeCurDist {
-      get { return kdTree.curdist; }
-      set { kdTree.curdist = value; }
-    }
-    [Storable]
-    public int KDTreeDebugCounter {
-      get { return kdTree.debugcounter; }
-      set { kdTree.debugcounter = value; }
-    }
-    [Storable]
-    public int[] KDTreeIdx {
-      get { return kdTree.idx; }
-      set { kdTree.idx = value; }
-    }
-    [Storable]
-    public int KDTreeKCur {
-      get { return kdTree.kcur; }
-      set { kdTree.kcur = value; }
-    }
-    [Storable]
-    public int KDTreeKNeeded {
-      get { return kdTree.kneeded; }
-      set { kdTree.kneeded = value; }
-    }
-    [Storable]
-    public int KDTreeN {
-      get { return kdTree.n; }
-      set { kdTree.n = value; }
-    }
-    [Storable]
-    public int[] KDTreeNodes {
-      get { return kdTree.nodes; }
-      set { kdTree.nodes = value; }
-    }
-    [Storable]
-    public int KDTreeNormType {
-      get { return kdTree.normtype; }
-      set { kdTree.normtype = value; }
-    }
-    [Storable]
-    public int KDTreeNX {
-      get { return kdTree.nx; }
-      set { kdTree.nx = value; }
-    }
-    [Storable]
-    public int KDTreeNY {
-      get { return kdTree.ny; }
-      set { kdTree.ny = value; }
-    }
-    [Storable]
-    public double[] KDTreeR {
-      get { return kdTree.r; }
-      set { kdTree.r = value; }
-    }
-    [Storable]
-    public double KDTreeRNeeded {
-      get { return kdTree.rneeded; }
-      set { kdTree.rneeded = value; }
-    }
-    [Storable]
-    public bool KDTreeSelfMatch {
-      get { return kdTree.selfmatch; }
-      set { kdTree.selfmatch = value; }
-    }
-    [Storable]
-    public double[] KDTreeSplits {
-      get { return kdTree.splits; }
-      set { kdTree.splits = value; }
-    }
-    [Storable]
-    public int[] KDTreeTags {
-      get { return kdTree.tags; }
-      set { kdTree.tags = value; }
-    }
-    [Storable]
-    public double[] KDTreeX {
-      get { return kdTree.x; }
-      set { kdTree.x = value; }
-    }
-    [Storable]
-    public double[,] KDTreeXY {
-      get { return kdTree.xy; }
-      set { kdTree.xy = value; }
-    }
-    #endregion
   }
 }

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs

@@ -41 +41 @@
     private const string NearestNeighbourRegressionModelResultName = "Nearest neighbour regression solution";
     private const string WeightsParameterName = "Weights";
-    private const string SelfMatchParameterName = "SelfMatch";
 
     #region parameter properties
     public IFixedValueParameter<IntValue> KParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
-    }
-    public IFixedValueParameter<BoolValue> SelfMatchParameter {
-      get { return (IFixedValueParameter<BoolValue>)Parameters[SelfMatchParameterName]; }
     }
     public IValueParameter<DoubleArray> WeightsParameter {
@@ -61 +57 @@
         else KParameter.Value.Value = value;
       }
-    }
-    public bool SelfMatch {
-      get { return SelfMatchParameter.Value.Value; }
-      set { SelfMatchParameter.Value.Value = value; }
     }
     public DoubleArray Weights {
@@ -81 +73 @@
       Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
       Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
-      Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       Problem = new RegressionProblem();
     }
@@ -91 +82 @@
       if (!Parameters.ContainsKey(WeightsParameterName)) {
         Parameters.Add(new OptionalValueParameter<DoubleArray>(WeightsParameterName, "Optional: use weights to specify individual scaling values for all features. If not set the weights are calculated automatically (each feature is scaled to unit variance)"));
-      }
-      if (!Parameters.ContainsKey(SelfMatchParameterName)) {
-        Parameters.Add(new FixedValueParameter<BoolValue>(SelfMatchParameterName, "Should we use equal points for classification?", new BoolValue(false)));
       }
       #endregion
@@ -106 +94 @@
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();
-      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K, SelfMatch, weights);
+      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K, weights);
       Results.Add(new Result(NearestNeighbourRegressionModelResultName, "The nearest neighbour regression solution.", solution));
     }
 
-    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k, double[] weights = null) {
       var clonedProblemData = (IRegressionProblemData)problemData.Clone();
-      return new NearestNeighbourRegressionSolution(Train(problemData, k, selfMatch, weights), clonedProblemData);
+      return new NearestNeighbourRegressionSolution(Train(problemData, k, weights), clonedProblemData);
     }
 
-    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k, bool selfMatch = false, double[] weights = null) {
+    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k, double[] weights = null) {
       return new NearestNeighbourModel(problemData.Dataset,
         problemData.TrainingIndices,
         k,
-        selfMatch,
         problemData.TargetVariable,
         problemData.AllowedInputVariables,

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs

@@ -209 +209 @@
         alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, out multiLayerPerceptron);
       } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+
       alglib.mlpreport rep;
-
       int info;
       // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs

@@ -20 +20 @@
 #endregion
 
+extern alias alglib_3_7;
 using System;
 using System.Collections.Generic;
@@ -32 +33 @@
   /// Represents a neural network model for regression and classification
   /// </summary>
-  [StorableType("AEB9B960-FCA6-4A6D-BD5F-27BCE9CC5BEA")]
+  [StorableType("DABDBD64-E93B-4F50-A343-C8A92C1C48A4")]
   [Item("NeuralNetworkModel", "Represents a neural network for regression and classification.")]
   public sealed class NeuralNetworkModel : ClassificationModel, INeuralNetworkModel {
 
     private object mlpLocker = new object();
+
+
+
     private alglib.multilayerperceptron multiLayerPerceptron;
+    [Storable]
+    private string SerializedMultiLayerPerceptron {
+      get { alglib.mlpserialize(multiLayerPerceptron, out var ser); return ser; }
+      set { if (value != null) alglib.mlpunserialize(value, out multiLayerPerceptron); }
+    }
 
     public override IEnumerable<string> VariablesUsedForPrediction {
@@ -48 +57 @@
     private double[] classValues;
     [StorableConstructor]
-    private NeuralNetworkModel(StorableConstructorFlag _) : base(_) {
-      multiLayerPerceptron = new alglib.multilayerperceptron();
-    }
+    private NeuralNetworkModel(StorableConstructorFlag _) : base(_) { }
     private NeuralNetworkModel(NeuralNetworkModel original, Cloner cloner)
       : base(original, cloner) {
-      multiLayerPerceptron = new alglib.multilayerperceptron();
-      multiLayerPerceptron.innerobj.chunks = (double[,])original.multiLayerPerceptron.innerobj.chunks.Clone();
-      multiLayerPerceptron.innerobj.columnmeans = (double[])original.multiLayerPerceptron.innerobj.columnmeans.Clone();
-      multiLayerPerceptron.innerobj.columnsigmas = (double[])original.multiLayerPerceptron.innerobj.columnsigmas.Clone();
-      multiLayerPerceptron.innerobj.derror = (double[])original.multiLayerPerceptron.innerobj.derror.Clone();
-      multiLayerPerceptron.innerobj.dfdnet = (double[])original.multiLayerPerceptron.innerobj.dfdnet.Clone();
-      multiLayerPerceptron.innerobj.neurons = (double[])original.multiLayerPerceptron.innerobj.neurons.Clone();
-      multiLayerPerceptron.innerobj.nwbuf = (double[])original.multiLayerPerceptron.innerobj.nwbuf.Clone();
-      multiLayerPerceptron.innerobj.structinfo = (int[])original.multiLayerPerceptron.innerobj.structinfo.Clone();
-      multiLayerPerceptron.innerobj.weights = (double[])original.multiLayerPerceptron.innerobj.weights.Clone();
-      multiLayerPerceptron.innerobj.x = (double[])original.multiLayerPerceptron.innerobj.x.Clone();
-      multiLayerPerceptron.innerobj.y = (double[])original.multiLayerPerceptron.innerobj.y.Clone();
+      if (original.multiLayerPerceptron != null)
+        multiLayerPerceptron = (alglib.multilayerperceptron)original.multiLayerPerceptron.make_copy();
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       if (original.classValues != null)
@@ -73 +70 @@
       this.name = ItemName;
       this.description = ItemDescription;
-      this.multiLayerPerceptron = multiLayerPerceptron;
+      this.multiLayerPerceptron = (alglib.multilayerperceptron)multiLayerPerceptron.make_copy();
       this.allowedInputVariables = allowedInputVariables.ToArray();
       if (classValues != null)
@@ -95 +92 @@
           x[column] = inputData[row, column];
         }
-        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-save!
+        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-safe!
         lock (mlpLocker) {
           alglib.mlpprocess(multiLayerPerceptron, x, ref y);
@@ -115 +112 @@
           x[column] = inputData[row, column];
         }
-        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-save!
+        // NOTE: mlpprocess changes data in multiLayerPerceptron and is therefore not thread-safe!
         lock (mlpLocker) {
           alglib.mlpprocess(multiLayerPerceptron, x, ref y);
@@ -156 +153 @@
       return new NeuralNetworkClassificationSolution(this, new ClassificationProblemData(problemData));
     }
-
-    #region persistence
-    [Storable]
-    private double[,] MultiLayerPerceptronChunks {
-      get {
-        return multiLayerPerceptron.innerobj.chunks;
-      }
-      set {
-        multiLayerPerceptron.innerobj.chunks = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronColumnMeans {
-      get {
-        return multiLayerPerceptron.innerobj.columnmeans;
-      }
-      set {
-        multiLayerPerceptron.innerobj.columnmeans = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronColumnSigmas {
-      get {
-        return multiLayerPerceptron.innerobj.columnsigmas;
-      }
-      set {
-        multiLayerPerceptron.innerobj.columnsigmas = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronDError {
-      get {
-        return multiLayerPerceptron.innerobj.derror;
-      }
-      set {
-        multiLayerPerceptron.innerobj.derror = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronDfdnet {
-      get {
-        return multiLayerPerceptron.innerobj.dfdnet;
-      }
-      set {
-        multiLayerPerceptron.innerobj.dfdnet = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronNeurons {
-      get {
-        return multiLayerPerceptron.innerobj.neurons;
-      }
-      set {
-        multiLayerPerceptron.innerobj.neurons = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronNwbuf {
-      get {
-        return multiLayerPerceptron.innerobj.nwbuf;
-      }
-      set {
-        multiLayerPerceptron.innerobj.nwbuf = value;
-      }
-    }
-    [Storable]
-    private int[] MultiLayerPerceptronStuctinfo {
-      get {
-        return multiLayerPerceptron.innerobj.structinfo;
-      }
-      set {
-        multiLayerPerceptron.innerobj.structinfo = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronWeights {
-      get {
-        return multiLayerPerceptron.innerobj.weights;
-      }
-      set {
-        multiLayerPerceptron.innerobj.weights = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronX {
-      get {
-        return multiLayerPerceptron.innerobj.x;
-      }
-      set {
-        multiLayerPerceptron.innerobj.x = value;
-      }
-    }
-    [Storable]
-    private double[] MultiLayerPerceptronY {
-      get {
-        return multiLayerPerceptron.innerobj.y;
-      }
-      set {
-        multiLayerPerceptron.innerobj.y = value;
-      }
-    }
-    #endregion
   }
 }

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs

@@ -186 +186 @@
       IEnumerable<int> rows = problemData.TrainingIndices;
       double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      int nRows = inputMatrix.GetLength(0);
       if (inputMatrix.ContainsNanOrInfinity())
         throw new NotSupportedException("Neural network regression does not support NaN or infinity values in the input dataset.");
@@ -197 +198 @@
         alglib.mlpcreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, out multiLayerPerceptron);
       } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
-      alglib.mlpreport rep;
-      int nRows = inputMatrix.GetLength(0);
 
       int info;
       // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
-      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
+      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out _);
       if (info != 2) throw new ArgumentException("Error in calculation of neural network regression solution");
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame

@@ -29 +29 @@
   [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.ALGLIB", "3.7.0")]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.17")]
   [PluginDependency("HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm", "3.3")] // for GBM
   [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs

@@ -207 +207 @@
         inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
       }
-
-      alglib.dfreport rep;
-      var dForest = RandomForestUtil.CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out rep);
+      
+      var dForest = RandomForestUtil.CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out var rep);
 
       rmsError = rep.rmserror;
@@ -216 +215 @@
       outOfBagRelClassificationError = rep.oobrelclserror;
 
-      return new RandomForestModelFull(dForest, problemData.TargetVariable, problemData.AllowedInputVariables, classValues);
+      return new RandomForestModelFull(dForest, nTrees, problemData.TargetVariable, problemData.AllowedInputVariables, classValues);
     }
     #endregion

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs

@@ -20 +20 @@
 #endregion
 
+extern alias alglib_3_7;
+
 using System;
 using System.Collections.Generic;
@@ -39 +41 @@
   public sealed class RandomForestModel : ClassificationModel, IRandomForestModel {
     // not persisted
-    private alglib.decisionforest randomForest;
-    private alglib.decisionforest RandomForest {
+    private alglib_3_7.alglib.decisionforest randomForest;
+    private alglib_3_7.alglib.decisionforest RandomForest {
       get {
         // recalculate lazily
@@ -74 +76 @@
     private RandomForestModel(StorableConstructorFlag _) : base(_) {
       // for backwards compatibility (loading old solutions)
-      randomForest = new alglib.decisionforest();
+      randomForest = new alglib_3_7.alglib.decisionforest();
     }
     private RandomForestModel(RandomForestModel original, Cloner cloner)
       : base(original, cloner) {
-      randomForest = new alglib.decisionforest();
+      randomForest = new alglib_3_7.alglib.decisionforest();
       randomForest.innerobj.bufsize = original.randomForest.innerobj.bufsize;
       randomForest.innerobj.nclasses = original.randomForest.innerobj.nclasses;
@@ -100 +102 @@
 
     // random forest models can only be created through the static factory methods CreateRegressionModel and CreateClassificationModel
-    private RandomForestModel(string targetVariable, alglib.decisionforest randomForest,
+    private RandomForestModel(string targetVariable, alglib_3_7.alglib.decisionforest randomForest,
       int seed, IDataAnalysisProblemData originalTrainingData,
       int nTrees, double r, double m, double[] classValues = null)
@@ -151 +153 @@
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(RandomForest, x, ref y);
+        alglib_3_7.alglib.dfprocess(RandomForest, x, ref y);
         yield return y[0];
       }
@@ -169 +171 @@
           x[column] = inputData[row, column];
         }
-        alglib.dforest.dfprocessraw(RandomForest.innerobj, x, ref ys);
+        alglib_3_7.alglib.dforest.dfprocessraw(RandomForest.innerobj, x, ref ys);
         yield return ys.VariancePop();
       }
@@ -187 +189 @@
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(randomForest, x, ref y);
+        alglib_3_7.alglib.dfprocess(randomForest, x, ref y);
         // find class for with the largest probability value
         int maxProbClassIndex = 0;
@@ -315 +317 @@
       double[,] inputMatrix = problemData.Dataset.ToArray(variables, trainingIndices);
 
-      alglib.dfreport rep;
-      var dForest = RandomForestUtil.CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, 1, out rep);
+      var dForest = RandomForestUtil.CreateRandomForestModelAlglib_3_7(seed, inputMatrix, nTrees, r, m, 1, out var rep);
 
       rmsError = rep.rmserror;
@@ -353 +354 @@
       }
 
-      alglib.dfreport rep;
-      var dForest = RandomForestUtil.CreateRandomForestModel(seed, inputMatrix, nTrees, r, m, nClasses, out rep);
+      var dForest = RandomForestUtil.CreateRandomForestModelAlglib_3_7(seed, inputMatrix, nTrees, r, m, nClasses, out var rep);
 
       rmsError = rep.rmserror;

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelFull.cs

@@ -31 +31 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  [StorableType("9C797DF0-1169-4381-A732-6DAB90802839")]
+  [StorableType("55412E08-DAD4-4C2E-9181-C142E7EA9474")]
   [Item("RandomForestModelFull", "Represents a random forest for regression and classification.")]
   public sealed class RandomForestModelFull : ClassificationModel, IRandomForestModel {
@@ -42 +42 @@
     private double[] classValues;
 
+    public int NumClasses => classValues == null ? 0 : classValues.Length;
+
     [Storable]
     private string[] inputVariables;
 
+    [Storable]
     public int NumberOfTrees {
-      get { return RandomForestNTrees; }
+      get; private set;
     }
 
@@ -53 +56 @@
 
     [Storable]
-    private int RandomForestBufSize {
-      get { return randomForest.innerobj.bufsize; }
-      set { randomForest.innerobj.bufsize = value; }
-    }
-    [Storable]
-    private int RandomForestNClasses {
-      get { return randomForest.innerobj.nclasses; }
-      set { randomForest.innerobj.nclasses = value; }
-    }
-    [Storable]
-    private int RandomForestNTrees {
-      get { return randomForest.innerobj.ntrees; }
-      set { randomForest.innerobj.ntrees = value; }
-    }
-    [Storable]
-    private int RandomForestNVars {
-      get { return randomForest.innerobj.nvars; }
-      set { randomForest.innerobj.nvars = value; }
-    }
-    [Storable]
-    private double[] RandomForestTrees {
-      get { return randomForest.innerobj.trees; }
-      set { randomForest.innerobj.trees = value; }
+    private string RandomForestSerialized {
+      get { alglib.dfserialize(randomForest, out var serialized); return serialized; }
+      set { if (value != null) alglib.dfunserialize(value, out randomForest); }
     }
 
     [StorableConstructor]
-    private RandomForestModelFull(StorableConstructorFlag _) : base(_) {
-      randomForest = new alglib.decisionforest();
-    }
+    private RandomForestModelFull(StorableConstructorFlag _) : base(_) { }
 
     private RandomForestModelFull(RandomForestModelFull original, Cloner cloner) : base(original, cloner) {
-      randomForest = new alglib.decisionforest();
-      randomForest.innerobj.bufsize = original.randomForest.innerobj.bufsize;
-      randomForest.innerobj.nclasses = original.randomForest.innerobj.nclasses;
-      randomForest.innerobj.ntrees = original.randomForest.innerobj.ntrees;
-      randomForest.innerobj.nvars = original.randomForest.innerobj.nvars;
-      randomForest.innerobj.trees = (double[])original.randomForest.innerobj.trees.Clone();
+      if (original.randomForest != null)
+        randomForest = (alglib.decisionforest)original.randomForest.make_copy();
+      NumberOfTrees = original.NumberOfTrees;
 
       // following fields are immutable so we don't need to clone them
@@ -99 +77 @@
     }
 
-    public RandomForestModelFull(alglib.decisionforest decisionForest, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<double> classValues = null) : base(targetVariable) {
+    public RandomForestModelFull(alglib.decisionforest decisionForest, int nTrees, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<double> classValues = null) : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
 
-      randomForest = decisionForest;
+      randomForest = (alglib.decisionforest)decisionForest.make_copy();
+      NumberOfTrees = nTrees;
 
       this.inputVariables = inputVariables.ToArray();
@@ -147 +126 @@
       double[] y = new double[1];
 
+      alglib.dfcreatebuffer(randomForest, out var buf);
       for (int row = 0; row < n; row++) {
         for (int column = 0; column < columns; column++) {
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(randomForest, x, ref y);
+        alglib.dftsprocess(randomForest, buf, x, ref y); // thread-safe process (as long as separate buffers are used)
         yield return y[0];
       }
@@ -168 +148 @@
           x[column] = inputData[row, column];
         }
-        alglib.dforest.dfprocessraw(randomForest.innerobj, x, ref ys);
+        lock (randomForest)
+          alglib.dforest.dfprocessraw(randomForest.innerobj, x, ref ys, null);
         yield return ys.VariancePop();
       }
@@ -180 +161 @@
       int columns = inputData.GetLength(1);
       double[] x = new double[columns];
-      double[] y = new double[randomForest.innerobj.nclasses];
-
+      double[] y = new double[NumClasses];
+
+      alglib.dfcreatebuffer(randomForest, out var buf);
       for (int row = 0; row < n; row++) {
         for (int column = 0; column < columns; column++) {
           x[column] = inputData[row, column];
         }
-        alglib.dfprocess(randomForest, x, ref y);
+        alglib.dftsprocess(randomForest, buf, x, ref y);
         // find class for with the largest probability value
         int maxProbClassIndex = 0;

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelSurrogate.cs

@@ -135 +135 @@
     }
 
+    public override bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
+      return ActualModel.IsProblemDataCompatible(problemData, out errorMessage);
+    }
+
     //RegressionModel methods
     public bool IsProblemDataCompatible(IRegressionProblemData problemData, out string errorMessage) {

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs

@@ -20 +20 @@
 #endregion
 
+extern alias alglib_3_7;
+using alglib_3_7;
 using System.Collections.Generic;
 using System.Linq;
@@ -200 +202 @@
       outOfBagAvgRelError = rep.oobavgrelerror;
 
-      return new RandomForestModelFull(dForest, problemData.TargetVariable, problemData.AllowedInputVariables);
+      return new RandomForestModelFull(dForest, nTrees, problemData.TargetVariable, problemData.AllowedInputVariables);
     }
 

branches/3106_AnalyticContinuedFractionsRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestUtil.cs

@@ -22 +22 @@
 #endregion
 
+extern alias alglib_3_7;
+
 using System;
 using System.Collections.Generic;
@@ -103 +105 @@
       RandomForestUtil.AssertInputMatrix(inputMatrix);
 
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+
+      alglib.dfbuildercreate(out var dfbuilder);
+      alglib.dfbuildersetdataset(dfbuilder, inputMatrix, nRows, nColumns - 1, nClasses);
+      alglib.dfbuildersetimportancenone(dfbuilder); // do not calculate importance (TODO add this feature)
+      alglib.dfbuildersetrdfalgo(dfbuilder, 0); // only one algorithm supported in version 3.17
+      alglib.dfbuildersetrdfsplitstrength(dfbuilder, 2); // 0 = split at the random position, fastest one
+                                                         // 1 = split at the middle of the range
+                                                         // 2 = strong split at the best point of the range (default)
+      alglib.dfbuildersetrndvarsratio(dfbuilder, m);
+      alglib.dfbuildersetsubsampleratio(dfbuilder, r);
+      alglib.dfbuildersetseed(dfbuilder, seed);
+      alglib.dfbuilderbuildrandomforest(dfbuilder, nTrees, out var dForest, out rep);
+      return dForest;
+    }
+    internal static alglib_3_7.alglib.decisionforest CreateRandomForestModelAlglib_3_7(int seed, double[,] inputMatrix, int nTrees, double r, double m, int nClasses, out alglib_3_7.alglib.dfreport rep) {
+      RandomForestUtil.AssertParameters(r, m);
+      RandomForestUtil.AssertInputMatrix(inputMatrix);
+
       int info = 0;
-      alglib.math.rndobject = new System.Random(seed);
-      var dForest = new alglib.decisionforest();
-      rep = new alglib.dfreport();
+      alglib_3_7.alglib.math.rndobject = new System.Random(seed);
+      var dForest = new alglib_3_7.alglib.decisionforest();
+      rep = new alglib_3_7.alglib.dfreport();
       int nRows = inputMatrix.GetLength(0);
       int nColumns = inputMatrix.GetLength(1);
@@ -112 +134 @@
       int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
 
-      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      alglib_3_7.alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib_3_7.alglib.dforest.dfusestrongsplits + alglib_3_7.alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
       if (info != 1) throw new ArgumentException("Error in calculation of random forest model");
       return dForest;
@@ -123 +145 @@
       var targetVariable = GetTargetVariableName(problemData);
       foreach (var tuple in partitions) {
-        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
         var trainingRandomForestPartition = tuple.Item1;
         var testRandomForestPartition = tuple.Item2;
-        var model = RandomForestModel.CreateRegressionModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var model = RandomForestRegression.CreateRandomForestRegressionModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed,
+                                                                             out var rmsError, out var avgRelError, out var outOfBagRmsError, out var outOfBagAvgRelError);
         var estimatedValues = model.GetEstimatedValues(ds, testRandomForestPartition);
         var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
@@ -143 +165 @@
       var targetVariable = GetTargetVariableName(problemData);
       foreach (var tuple in partitions) {
-        double rmsError, avgRelError, outOfBagAvgRelError, outOfBagRmsError;
         var trainingRandomForestPartition = tuple.Item1;
         var testRandomForestPartition = tuple.Item2;
-        var model = RandomForestModel.CreateClassificationModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+        var model = RandomForestClassification.CreateRandomForestClassificationModel(problemData, trainingRandomForestPartition, nTrees, r, m, seed,
+                                                                                     out var rmsError, out var avgRelError, out var outOfBagRmsError, out var outOfBagAvgRelError);
         var estimatedValues = model.GetEstimatedClassValues(ds, testRandomForestPartition);
         var targetValues = ds.GetDoubleValues(targetVariable, testRandomForestPartition);
@@ -176 +198 @@
         var parameters = new RFParameter();
         for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
-        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
-        RandomForestModel.CreateRegressionModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed, out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+        RandomForestRegression.CreateRandomForestRegressionModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
+                                                                 out var rmsError, out var avgRelError, out var outOfBagRmsError, out var outOfBagAvgRelError);
 
         lock (locker) {
@@ -208 +230 @@
         var parameters = new RFParameter();
         for (int i = 0; i < setters.Count; ++i) { setters[i](parameters, parameterValues[i]); }
-        double rmsError, outOfBagRmsError, avgRelError, outOfBagAvgRelError;
-        RandomForestModel.CreateClassificationModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
-                                                                out rmsError, out outOfBagRmsError, out avgRelError, out outOfBagAvgRelError);
+        RandomForestClassification.CreateRandomForestClassificationModel(problemData, problemData.TrainingIndices, parameters.N, parameters.R, parameters.M, seed,
+                                                                         out var rmsError, out var avgRelError, out var outOfBagRmsError, out var outOfBagAvgRelError);
 
         lock (locker) {
@@ -227 +248 @@
     /// <param name="problemData">The regression problem data</param>
     /// <param name="numberOfFolds">The number of folds for crossvalidation</param>
-    /// <param name="shuffleFolds">Specifies whether the folds should be shuffled</param>
     /// <param name="parameterRanges">The ranges for each parameter in the grid search</param>
     /// <param name="seed">The random seed (required by the random forest model)</param>
     /// <param name="maxDegreeOfParallelism">The maximum allowed number of threads (to parallelize the grid search)</param>
     /// <returns>The best parameter values found by the grid search</returns>
-    public static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, bool shuffleFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
+    public static RFParameter GridSearch(IRegressionProblemData problemData, int numberOfFolds, Dictionary<string, IEnumerable<double>> parameterRanges, int seed = 12345, int maxDegreeOfParallelism = 1) {
       DoubleValue mse = new DoubleValue(Double.MaxValue);
       RFParameter bestParameter = new RFParameter();