Changeset 14869

Timestamp:

04/14/17 08:58:45 (8 years ago)

Author:

gkronber

Message:

#2699: merged changesets from trunk to branch

Location:

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• . (modified) (1 prop)
• BaselineClassifiers/OneFactorClassificationModel.cs (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs)
• BaselineClassifiers/OneFactorClassificationSolution.cs (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs)
• BaselineClassifiers/OneR.cs (modified) (7 diffs)
• BaselineClassifiers/OneRClassificationModel.cs (modified) (6 diffs)
• BaselineClassifiers/OneRClassificationSolution.cs (modified) (2 diffs)
• BaselineClassifiers/ZeroR.cs (modified) (2 diffs)
• CrossValidation.cs (modified) (1 diff)
• FixedDataAnalysisAlgorithm.cs (modified) (3 diffs)
• GBM/GradientBoostingRegressionAlgorithm.cs (modified) (1 diff)
• GaussianProcess/GaussianProcessBase.cs (modified) (4 diffs)
• GaussianProcess/GaussianProcessClassificationModelCreator.cs (modified) (1 diff)
• GaussianProcess/GaussianProcessModel.cs (modified) (4 diffs)
• GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs (modified) (4 diffs)
• GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (3 diffs)
• Linear/AlglibUtil.cs (deleted)
• Linear/LinearDiscriminantAnalysis.cs (modified) (5 diffs)
• Linear/LinearRegression.cs (modified) (4 diffs)
• Linear/MultinomialLogitClassification.cs (modified) (4 diffs)
• Linear/MultinomialLogitClassificationSolution.cs (modified) (1 diff)
• Linear/MultinomialLogitModel.cs (modified) (3 diffs)
• Linear/Scaling.cs (modified) (1 diff)
• MctsSymbolicRegression/MctsSymbolicRegressionAlgorithm.cs (modified) (1 diff)
• Nca/Initialization/LdaInitializer.cs (modified) (1 diff)
• Nca/Initialization/PcaInitializer.cs (modified) (1 diff)
• Nca/NcaGradientCalculator.cs (modified) (1 diff)
• Nca/NcaModel.cs (modified) (1 diff)
• NearestNeighbour/NearestNeighbourClassification.cs (modified) (2 diffs)
• NearestNeighbour/NearestNeighbourModel.cs (modified) (5 diffs)
• NearestNeighbour/NearestNeighbourRegression.cs (modified) (2 diffs)
• NeuralNetwork/NeuralNetworkClassification.cs (modified) (3 diffs)
• NeuralNetwork/NeuralNetworkEnsembleClassification.cs (modified) (4 diffs)
• NeuralNetwork/NeuralNetworkEnsembleModel.cs (modified) (2 diffs)
• NeuralNetwork/NeuralNetworkEnsembleRegression.cs (modified) (3 diffs)
• NeuralNetwork/NeuralNetworkModel.cs (modified) (2 diffs)
• NeuralNetwork/NeuralNetworkRegression.cs (modified) (3 diffs)
• NonlinearRegression/NonlinearRegression.cs (modified) (3 diffs)
• RandomForest/RandomForestClassification.cs (modified) (2 diffs)
• RandomForest/RandomForestModel.cs (modified) (5 diffs)
• RandomForest/RandomForestRegression.cs (modified) (2 diffs)
• SupportVectorMachine/SupportVectorClassification.cs (modified) (2 diffs)
• SupportVectorMachine/SupportVectorRegression.cs (modified) (2 diffs)
• TimeSeries/AutoregressiveModeling.cs (modified) (3 diffs)
• kMeans/KMeansClustering.cs (modified) (3 diffs)

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -58 +60 @@
     }
 
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       var solution = CreateOneRSolution(Problem.ProblemData, MinBucketSizeParameter.Value.Value);
       Results.Add(new Result("OneR solution", "The 1R classifier.", solution));
@@ -64 +66 @@
 
     public static IClassificationSolution CreateOneRSolution(IClassificationProblemData problemData, int minBucketSize = 6) {
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var model1 = FindBestDoubleVariableModel(problemData, minBucketSize);
+      var model2 = FindBestFactorModel(problemData);
+
+      if (model1 == null && model2 == null) throw new InvalidProgramException("Could not create OneR solution");
+      else if (model1 == null) return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+      else if (model2 == null) return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+      else {
+        var model1EstimatedValues = model1.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var model1NumCorrect = classValues.Zip(model1EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+        var model2EstimatedValues = model2.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var model2NumCorrect = classValues.Zip(model2EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+        if (model1NumCorrect > model2NumCorrect) {
+          return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+        } else {
+          return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+        }
+      }
+    }
+
+    private static OneRClassificationModel FindBestDoubleVariableModel(IClassificationProblemData problemData, int minBucketSize = 6) {
       var bestClassified = 0;
       List<Split> bestSplits = null;
@@ -70 +95 @@
       var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
 
-      foreach (var variable in problemData.AllowedInputVariables) {
+      var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType<double>);
+
+      if (!allowedInputVariables.Any()) return null;
+
+      foreach (var variable in allowedInputVariables) {
         var inputValues = problemData.Dataset.GetDoubleValues(variable, problemData.TrainingIndices);
         var samples = inputValues.Zip(classValues, (i, v) => new Sample(i, v)).OrderBy(s => s.inputValue);
 
-        var missingValuesDistribution = samples.Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue).ToDictionary(s => s.Key, s => s.Count()).MaxItems(s => s.Value).FirstOrDefault();
+        var missingValuesDistribution = samples
+          .Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue)
+          .ToDictionary(s => s.Key, s => s.Count())
+          .MaxItems(s => s.Value)
+          .FirstOrDefault();
 
         //calculate class distributions for all distinct inputValues
@@ -119 +152 @@
           while (sample.inputValue >= splits[splitIndex].thresholdValue)
             splitIndex++;
-          correctClassified += sample.classValue == splits[splitIndex].classValue ? 1 : 0;
+          correctClassified += sample.classValue.IsAlmost(splits[splitIndex].classValue) ? 1 : 0;
         }
         correctClassified += missingValuesDistribution.Value;
@@ -133 +166 @@
       //remove neighboring splits with the same class value
       for (int i = 0; i < bestSplits.Count - 1; i++) {
-        if (bestSplits[i].classValue == bestSplits[i + 1].classValue) {
+        if (bestSplits[i].classValue.IsAlmost(bestSplits[i + 1].classValue)) {
           bestSplits.Remove(bestSplits[i]);
           i--;
@@ -139 +172 @@
       }
 
-      var model = new OneRClassificationModel(problemData.TargetVariable, bestVariable, bestSplits.Select(s => s.thresholdValue).ToArray(), bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
-      var solution = new OneRClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
-
-      return solution;
+      var model = new OneRClassificationModel(problemData.TargetVariable, bestVariable,
+        bestSplits.Select(s => s.thresholdValue).ToArray(),
+        bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
+
+      return model;
+    }
+    private static OneFactorClassificationModel FindBestFactorModel(IClassificationProblemData problemData) {
+      var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+      var defaultClass = FindMostFrequentClassValue(classValues);
+      // only select string variables
+      var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType<string>);
+
+      if (!allowedInputVariables.Any()) return null;
+
+      OneFactorClassificationModel bestModel = null;
+      var bestModelNumCorrect = 0;
+
+      foreach (var variable in allowedInputVariables) {
+        var variableValues = problemData.Dataset.GetStringValues(variable, problemData.TrainingIndices);
+        var groupedClassValues = variableValues
+          .Zip(classValues, (v, c) => new KeyValuePair<string, double>(v, c))
+          .GroupBy(kvp => kvp.Key)
+          .ToDictionary(g => g.Key, g => FindMostFrequentClassValue(g.Select(kvp => kvp.Value)));
+
+        var model = new OneFactorClassificationModel(problemData.TargetVariable, variable,
+          groupedClassValues.Select(kvp => kvp.Key).ToArray(), groupedClassValues.Select(kvp => kvp.Value).ToArray(), defaultClass);
+
+        var modelEstimatedValues = model.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+        var modelNumCorrect = classValues.Zip(modelEstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+        if (modelNumCorrect > bestModelNumCorrect) {
+          bestModelNumCorrect = modelNumCorrect;
+          bestModel = model;
+        }
+      }
+
+      return bestModel;
+    }
+
+    private static double FindMostFrequentClassValue(IEnumerable<double> classValues) {
+      return classValues.GroupBy(c => c).OrderByDescending(g => g.Count()).Select(g => g.Key).First();
     }
 

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs

@@ -31 +31 @@
   [StorableClass]
   [Item("OneR Classification Model", "A model that uses intervals for one variable to determine the class.")]
-  public class OneRClassificationModel : ClassificationModel {
+  public sealed class OneRClassificationModel : ClassificationModel {
     public override IEnumerable<string> VariablesUsedForPrediction {
       get { return new[] { Variable }; }
@@ -37 +37 @@
 
     [Storable]
-    protected string variable;
+    private string variable;
     public string Variable {
       get { return variable; }
@@ -43 +43 @@
 
     [Storable]
-    protected double[] splits;
+    private double[] splits;
     public double[] Splits {
       get { return splits; }
@@ -49 +49 @@
 
     [Storable]
-    protected double[] classes;
+    private double[] classes;
     public double[] Classes {
       get { return classes; }
@@ -55 +55 @@
 
     [Storable]
-    protected double missingValuesClass;
+    private double missingValuesClass;
     public double MissingValuesClass {
       get { return missingValuesClass; }
@@ -61 +61 @@
 
     [StorableConstructor]
-    protected OneRClassificationModel(bool deserializing) : base(deserializing) { }
-    protected OneRClassificationModel(OneRClassificationModel original, Cloner cloner)
+    private OneRClassificationModel(bool deserializing) : base(deserializing) { }
+    private OneRClassificationModel(OneRClassificationModel original, Cloner cloner)
       : base(original, cloner) {
       this.variable = (string)original.variable;
       this.splits = (double[])original.splits.Clone();
       this.classes = (double[])original.classes.Clone();
+      this.missingValuesClass = original.missingValuesClass;
     }
     public override IDeepCloneable Clone(Cloner cloner) { return new OneRClassificationModel(this, cloner); }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs

@@ -28 +28 @@
   [StorableClass]
   [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single feature with potentially multiple thresholds for class prediction.")]
-  public class OneRClassificationSolution : ClassificationSolution {
+  public sealed class OneRClassificationSolution : ClassificationSolution {
     public new OneRClassificationModel Model {
       get { return (OneRClassificationModel)base.Model; }
@@ -35 +35 @@
 
     [StorableConstructor]
-    protected OneRClassificationSolution(bool deserializing) : base(deserializing) { }
-    protected OneRClassificationSolution(OneRClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+    private OneRClassificationSolution(bool deserializing) : base(deserializing) { }
+    private OneRClassificationSolution(OneRClassificationSolution original, Cloner cloner) : base(original, cloner) { }
     public OneRClassificationSolution(OneRClassificationModel model, IClassificationProblemData problemData)
       : base(model, problemData) {

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs

@@ -21 +21 @@
 
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -49 +50 @@
     }
 
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       var solution = CreateZeroRSolution(Problem.ProblemData);
       Results.Add(new Result("ZeroR solution", "The simplest possible classifier, ZeroR always predicts the majority class.", solution));

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

@@ -451 +451 @@
       var aggregatedResults = new List<IResult>();
       foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
-        // clone manually to correctly clone references between cloned root objects 
-        Cloner cloner = new Cloner();
-        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
+        // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
+        var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
+        var problemDataClone = new ClassificationProblemData(Problem.ProblemData.Dataset,
+          Problem.ProblemData.AllowedInputVariables, targetVariable);
         // set partitions of problem data clone correctly
         problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs

@@ -21 +21 @@
 
 using System;
-using System.Threading;
-using System.Threading.Tasks;
 using HeuristicLab.Common;
 using HeuristicLab.Optimization;
@@ -30 +28 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  public abstract class FixedDataAnalysisAlgorithm<T> : Algorithm,
-    IDataAnalysisAlgorithm<T>,
-    IStorableContent
-    where T : class, IDataAnalysisProblem {
-    public string Filename { get; set; }
-
+  public abstract class FixedDataAnalysisAlgorithm<T> : BasicAlgorithm where T : class, IDataAnalysisProblem {
     #region Properties
     public override Type ProblemType {
@@ -44 +37 @@
       set { base.Problem = value; }
     }
-    [Storable]
-    private ResultCollection results;
-    public override ResultCollection Results {
-      get { return results; }
-    }
     #endregion
 
-    private DateTime lastUpdateTime;
+    public override bool SupportsPause { get { return false; } }
 
     [StorableConstructor]
     protected FixedDataAnalysisAlgorithm(bool deserializing) : base(deserializing) { }
-    protected FixedDataAnalysisAlgorithm(FixedDataAnalysisAlgorithm<T> original, Cloner cloner)
-      : base(original, cloner) {
-      results = cloner.Clone(original.Results);
-    }
-    public FixedDataAnalysisAlgorithm()
-      : base() {
-      results = new ResultCollection();
-    }
-
-    public override void Prepare() {
-      if (Problem != null) base.Prepare();
-      results.Clear();
-      OnPrepared();
-    }
-
-    public override void Start() {
-      base.Start();
-      var cancellationTokenSource = new CancellationTokenSource();
-
-      OnStarted();
-      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
-      task.ContinueWith(t => {
-        try {
-          t.Wait();
-        }
-        catch (AggregateException ex) {
-          try {
-            ex.Flatten().Handle(x => x is OperationCanceledException);
-          }
-          catch (AggregateException remaining) {
-            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
-            else OnExceptionOccurred(remaining);
-          }
-        }
-        cancellationTokenSource.Dispose();
-        cancellationTokenSource = null;
-        OnStopped();
-      });
-    }
-    private void Run(object state) {
-      CancellationToken cancellationToken = (CancellationToken)state;
-      lastUpdateTime = DateTime.UtcNow;
-      System.Timers.Timer timer = new System.Timers.Timer(250);
-      timer.AutoReset = true;
-      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
-      timer.Start();
-      try {
-        Run();
-      }
-      finally {
-        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
-        timer.Stop();
-        ExecutionTime += DateTime.UtcNow - lastUpdateTime;
-      }
-
-      cancellationToken.ThrowIfCancellationRequested();
-    }
-    protected abstract void Run();
-    #region Events
-    protected override void OnProblemChanged() {
-      Problem.Reset += new EventHandler(Problem_Reset);
-      base.OnProblemChanged();
-    }
-    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
-      System.Timers.Timer timer = (System.Timers.Timer)sender;
-      timer.Enabled = false;
-      DateTime now = DateTime.UtcNow;
-      ExecutionTime += now - lastUpdateTime;
-      lastUpdateTime = now;
-      timer.Enabled = true;
-    }
-    #endregion
+    protected FixedDataAnalysisAlgorithm(FixedDataAnalysisAlgorithm<T> original, Cloner cloner) : base(original, cloner) { }
+    public FixedDataAnalysisAlgorithm() : base() { }
 
   }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs

@@ -44 +44 @@
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 350)]
-  public class GradientBoostingRegressionAlgorithm : BasicAlgorithm {
-    public override Type ProblemType {
-      get { return typeof(IRegressionProblem); }
-    }
-
-    public new IRegressionProblem Problem {
-      get { return (IRegressionProblem)base.Problem; }
-      set { base.Problem = value; }
-    }
+  public class GradientBoostingRegressionAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
 
     #region ParameterNames

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessBase.cs

@@ -21 +21 @@
 #endregion
 
+using System.Linq;
 using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
@@ -119 +120 @@
 
       // necessary for BFGS
-      Parameters.Add(new ValueParameter<BoolValue>("Maximization", new BoolValue(false)));
-      Parameters["Maximization"].Hidden = true;
+      Parameters.Add(new FixedValueParameter<BoolValue>("Maximization (BFGS)", new BoolValue(false)));
+      Parameters["Maximization (BFGS)"].Hidden = true;
 
       var randomCreator = new HeuristicLab.Random.RandomCreator();
@@ -164 +165 @@
       modelCreator.Successor = updateResults;
 
+      updateResults.MaximizationParameter.ActualName = "Maximization (BFGS)";
       updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
       updateResults.QualityParameter.ActualName = NegativeLogLikelihoodParameterName;
@@ -197 +199 @@
       // BackwardsCompatibility3.4
       #region Backwards compatible code, remove with 3.5
-      if (!Parameters.ContainsKey("Maximization")) {
-        Parameters.Add(new ValueParameter<BoolValue>("Maximization", new BoolValue(false)));
-        Parameters["Maximization"].Hidden = true;
+      if (Parameters.ContainsKey("Maximization")) {
+        Parameters.Remove("Maximization");
+      }
+
+      if (!Parameters.ContainsKey("Maximization (BFGS)")) {
+        Parameters.Add(new FixedValueParameter<BoolValue>("Maximization (BFGS)", new BoolValue(false)));
+        Parameters["Maximization (BFGS)"].Hidden = true;
+        OperatorGraph.Operators.OfType<LbfgsUpdateResults>().First().MaximizationParameter.ActualName = "Maximization BFGS";
       }
 

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs

@@ -67 +67 @@
         HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
         return base.Apply();
-      } catch (ArgumentException) { } catch (alglib.alglibexception) { }
+      } catch (ArgumentException) {
+      } catch (alglib.alglibexception) {
+      }
       NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
       HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

@@ -165 +165 @@
       try {
         CalculateModel(ds, rows, scaleInputs);
-      }
-      catch (alglib.alglibexception ae) {
+      } catch (alglib.alglibexception ae) {
         // wrap exception so that calling code doesn't have to know about alglib implementation
         throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
@@ -260 +259 @@
     private static double[,] GetData(IDataset ds, IEnumerable<string> allowedInputs, IEnumerable<int> rows, Scaling scaling) {
       if (scaling != null) {
-        return AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputs, rows, scaling);
+        // BackwardsCompatibility3.3
+        #region Backwards compatible code, remove with 3.4
+        // TODO: completely remove Scaling class
+        List<string> variablesList = allowedInputs.ToList();
+        List<int> rowsList = rows.ToList();
+
+        double[,] matrix = new double[rowsList.Count, variablesList.Count];
+
+        int col = 0;
+        foreach (string column in variablesList) {
+          var values = scaling.GetScaledValues(ds, column, rowsList);
+          int row = 0;
+          foreach (var value in values) {
+            matrix[row, col] = value;
+            row++;
+          }
+          col++;
+        }
+        return matrix;
+        #endregion
       } else {
-        return AlglibUtil.PrepareInputMatrix(ds, allowedInputs, rows);
+        return ds.ToArray(allowedInputs, rows);
       }
     }
@@ -334 +352 @@
         return Enumerable.Range(0, newN)
           .Select(i => ms[i] + Util.ScalarProd(Ks[i], alpha));
-      }
-      catch (alglib.alglibexception ae) {
+      } catch (alglib.alglibexception ae) {
         // wrap exception so that calling code doesn't have to know about alglib implementation
         throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);
@@ -381 +398 @@
         }
         return kss;
-      }
-      catch (alglib.alglibexception ae) {
+      } catch (alglib.alglibexception ae) {
         // wrap exception so that calling code doesn't have to know about alglib implementation
         throw new ArgumentException("There was a problem in the calculation of the Gaussian process model", ae);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithm.cs

@@ -21 +21 @@
 #endregion
 
-using System;
 using System.Linq;
 using System.Threading;
@@ -38 +37 @@
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 125)]
-  public class GradientBoostedTreesAlgorithm : BasicAlgorithm {
-    public override Type ProblemType {
-      get { return typeof(IRegressionProblem); }
-    }
-    public new IRegressionProblem Problem {
-      get { return (IRegressionProblem)base.Problem; }
-      set { base.Problem = value; }
-    }
-
+  public class GradientBoostedTreesAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
     #region ParameterNames
     private const string IterationsParameterName = "Iterations";
@@ -204 +195 @@
       table.Rows.Add(new DataRow("Loss (train)"));
       table.Rows.Add(new DataRow("Loss (test)"));
+      table.Rows["Loss (train)"].VisualProperties.StartIndexZero = true;
+      table.Rows["Loss (test)"].VisualProperties.StartIndexZero = true;
+
       Results.Add(new Result("Qualities", table));
       var curLoss = new DoubleValue();
@@ -263 +257 @@
           var classificationProblemData = new ClassificationProblemData(problemData.Dataset,
             problemData.AllowedInputVariables, problemData.TargetVariable, problemData.Transformations);
-          classificationModel.RecalculateModelParameters(classificationProblemData, classificationProblemData.TrainingIndices);
+          classificationProblemData.TrainingPartition.Start = Problem.ProblemData.TrainingPartition.Start;
+          classificationProblemData.TrainingPartition.End = Problem.ProblemData.TrainingPartition.End;
+          classificationProblemData.TestPartition.Start = Problem.ProblemData.TestPartition.Start;
+          classificationProblemData.TestPartition.End = Problem.ProblemData.TestPartition.End;
+
+          classificationModel.SetThresholdsAndClassValues(new double[] { double.NegativeInfinity, 0.0 }, new[] { 0.0, 1.0 });
+
 
           var classificationSolution = new DiscriminantFunctionClassificationSolution(classificationModel, classificationProblemData);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs

@@ -148 +148 @@
     // for custom stepping & termination
     public static IGbmState CreateGbmState(IRegressionProblemData problemData, ILossFunction lossFunction, uint randSeed, int maxSize = 3, double r = 0.66, double m = 0.5, double nu = 0.01) {
+      // check input variables. Only double variables are allowed.
+      var invalidInputs =
+        problemData.AllowedInputVariables.Where(name => !problemData.Dataset.VariableHasType<double>(name));
+      if (invalidInputs.Any())
+        throw new NotSupportedException("Gradient tree boosting only supports real-valued variables. Unsupported inputs: " + string.Join(", ", invalidInputs));
+
       return new GbmState(problemData, lossFunction, randSeed, maxSize, r, m, nu);
     }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -189 +189 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="BaselineClassifiers\OneFactorClassificationModel.cs" />
+    <Compile Include="BaselineClassifiers\OneFactorClassificationSolution.cs" />
     <Compile Include="BaselineClassifiers\OneR.cs" />
     <Compile Include="BaselineClassifiers\OneRClassificationModel.cs" />
@@ -311 +313 @@
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="Linear\AlglibUtil.cs" />
-    <Compile Include="Linear\Scaling.cs" />
     <Compile Include="Linear\LinearDiscriminantAnalysis.cs" />
     <Compile Include="Linear\LinearRegression.cs">
@@ -320 +320 @@
     <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
     <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="Linear\Scaling.cs" />
     <Compile Include="MctsSymbolicRegression\Automaton.cs" />
     <Compile Include="MctsSymbolicRegression\CodeGenerator.cs" />

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36 +37 @@
   /// Linear discriminant analysis classification algorithm.
   /// </summary>
-  [Item("Linear Discriminant Analysis", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
+  [Item("Linear Discriminant Analysis (LDA)", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 100)]
   [StorableClass]
@@ -59 +60 @@
 
     #region Fisher LDA
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       var solution = CreateLinearDiscriminantAnalysisSolution(Problem.ProblemData);
       Results.Add(new Result(LinearDiscriminantAnalysisSolutionResultName, "The linear discriminant analysis.", solution));
@@ -70 +71 @@
       IEnumerable<int> rows = problemData.TrainingIndices;
       int nClasses = problemData.ClassNames.Count();
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      var doubleVariableNames = allowedInputVariables.Where(dataset.VariableHasType<double>).ToArray();
+      var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>).ToArray();
+      double[,] inputMatrix = dataset.ToArray(doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariables = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      var factorMatrix = dataset.ToArray(factorVariables, rows);
+
+      inputMatrix = factorMatrix.HorzCat(inputMatrix);
+
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Linear discriminant analysis does not support NaN or infinity values in the input dataset.");
@@ -82 +91 @@
       int info;
       double[] w;
-      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), allowedInputVariables.Count(), nClasses, out info, out w);
+      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), inputMatrix.GetLength(1) - 1, nClasses, out info, out w);
       if (info < 1) throw new ArgumentException("Error in calculation of linear discriminant analysis solution");
 
-      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
-      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
-      tree.Root.AddSubtree(startNode);
-      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
-      startNode.AddSubtree(addition);
+      var nFactorCoeff = factorMatrix.GetLength(1);
+      var tree = LinearModelToTreeConverter.CreateTree(factorVariables, w.Take(nFactorCoeff).ToArray(),
+        doubleVariableNames, w.Skip(nFactorCoeff).Take(doubleVariableNames.Length).ToArray());
 
-      int col = 0;
-      foreach (string column in allowedInputVariables) {
-        VariableTreeNode vNode = (VariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
-        vNode.VariableName = column;
-        vNode.Weight = w[col];
-        addition.AddSubtree(vNode);
-        col++;
-      }
-
-      var model = LinearDiscriminantAnalysis.CreateDiscriminantFunctionModel(tree, new SymbolicDataAnalysisExpressionTreeInterpreter(), problemData, rows);
+      var model = CreateDiscriminantFunctionModel(tree, new SymbolicDataAnalysisExpressionTreeLinearInterpreter(), problemData, rows);
       SymbolicDiscriminantFunctionClassificationSolution solution = new SymbolicDiscriminantFunctionClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
 

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
@@ -60 +60 @@
 
     #region linear regression
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, cvRmsError;
       var solution = CreateLinearRegressionSolution(Problem.ProblemData, out rmsError, out cvRmsError);
@@ -73 +73 @@
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      var doubleVariables = allowedInputVariables.Where(dataset.VariableHasType<double>);
+      var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>);
+      var factorVariables = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      double[,] binaryMatrix = dataset.ToArray(factorVariables, rows);
+      double[,] doubleVarMatrix = dataset.ToArray(doubleVariables.Concat(new string[] { targetVariable }), rows);
+      var inputMatrix = binaryMatrix.HorzCat(doubleVarMatrix);
+
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Linear regression does not support NaN or infinity values in the input dataset.");
@@ -91 +97 @@
       alglib.lrunpack(lm, out coefficients, out nFeatures);
 
-      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
-      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
-      tree.Root.AddSubtree(startNode);
-      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
-      startNode.AddSubtree(addition);
-
-      int col = 0;
-      foreach (string column in allowedInputVariables) {
-        VariableTreeNode vNode = (VariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
-        vNode.VariableName = column;
-        vNode.Weight = coefficients[col];
-        addition.AddSubtree(vNode);
-        col++;
-      }
-
-      ConstantTreeNode cNode = (ConstantTreeNode)new Constant().CreateTreeNode();
-      cNode.Value = coefficients[coefficients.Length - 1];
-      addition.AddSubtree(cNode);
-
-      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(problemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeInterpreter()), (IRegressionProblemData)problemData.Clone());
+      int nFactorCoeff = binaryMatrix.GetLength(1);
+      int nVarCoeff = doubleVariables.Count();
+      var tree = LinearModelToTreeConverter.CreateTree(factorVariables, coefficients.Take(nFactorCoeff).ToArray(),
+        doubleVariables.ToArray(), coefficients.Skip(nFactorCoeff).Take(nVarCoeff).ToArray(), 
+        @const: coefficients[nFeatures]);
+      
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(problemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeLinearInterpreter()), (IRegressionProblemData)problemData.Clone());
       solution.Model.Name = "Linear Regression Model";
       solution.Name = "Linear Regression Solution";

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -57 +58 @@
 
     #region logit classification
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, relClassError;
       var solution = CreateLogitClassificationSolution(Problem.ProblemData, out rmsError, out relClassError);
@@ -68 +69 @@
       var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      var doubleVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<double>);
+      var factorVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<string>);
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = dataset.ToArray(doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariableValues = dataset.GetFactorVariableValues(factorVariableNames, rows);
+      var factorMatrix = dataset.ToArray(factorVariableValues, rows);
+      inputMatrix = factorMatrix.HorzCat(inputMatrix);
+
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Multinomial logit classification does not support NaN or infinity values in the input dataset.");
@@ -95 +102 @@
       relClassError = alglib.mnlrelclserror(lm, inputMatrix, nRows);
 
-      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, allowedInputVariables, classValues), (IClassificationProblemData)problemData.Clone());
+      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, doubleVariableNames, factorVariableValues, classValues), (IClassificationProblemData)problemData.Clone());
       return solution;
     }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs

@@ -43 +43 @@
       : base(original, cloner) {
     }
-    public MultinomialLogitClassificationSolution( MultinomialLogitModel logitModel,IClassificationProblemData problemData)
+    public MultinomialLogitClassificationSolution(MultinomialLogitModel logitModel, IClassificationProblemData problemData)
       : base(logitModel, problemData) {
     }

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs

@@ -56 +56 @@
     [Storable]
     private double[] classValues;
+    [Storable]
+    private List<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+
     [StorableConstructor]
     private MultinomialLogitModel(bool deserializing)
@@ -68 +71 @@
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       classValues = (double[])original.classValues.Clone();
+      this.factorVariables = original.factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
     }
-    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
+    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> doubleInputVariables, IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables, double[] classValues)
       : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.logitModel = logitModel;
-      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.allowedInputVariables = doubleInputVariables.ToArray();
+      this.factorVariables = factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
       this.classValues = (double[])classValues.Clone();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      factorVariables = new List<KeyValuePair<string, IEnumerable<string>>>();
+      #endregion
     }
 
@@ -83 +96 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
+      double[,] factorData = dataset.ToArray(factorVariables, rows);
+
+      inputData = factorData.HorzCat(inputData);
 
       int n = inputData.GetLength(0);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs

@@ -29 +29 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Obsolete("Use transformation classes in Problems.DataAnalysis instead")]
   [StorableClass]
   [Item(Name = "Scaling", Description = "Contains information about scaling of variables for data-analysis algorithms.")]

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/MctsSymbolicRegression/MctsSymbolicRegressionAlgorithm.cs

@@ -38 +38 @@
   [StorableClass]
   [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 250)]
-  public class MctsSymbolicRegressionAlgorithm : BasicAlgorithm {
-    public override Type ProblemType {
-      get { return typeof(IRegressionProblem); }
-    }
-    public new IRegressionProblem Problem {
-      get { return (IRegressionProblem)base.Problem; }
-      set { base.Problem = value; }
-    }
+  public class MctsSymbolicRegressionAlgorithm : FixedDataAnalysisAlgorithm<IRegressionProblem> {
 
     #region ParameterNames

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs

@@ -44 +44 @@
       var attributes = data.AllowedInputVariables.Count();
 
-      var ldaDs = AlglibUtil.PrepareInputMatrix(data.Dataset,
-                                                data.AllowedInputVariables.Concat(data.TargetVariable.ToEnumerable()),
-                                                data.TrainingIndices);
+      var ldaDs = data.Dataset.ToArray(
+                                       data.AllowedInputVariables.Concat(data.TargetVariable.ToEnumerable()),
+                                       data.TrainingIndices);
 
       // map class values to sequential natural numbers (required by alglib)

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs

@@ -44 +44 @@
       var attributes = data.AllowedInputVariables.Count();
 
-      var pcaDs = AlglibUtil.PrepareInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices);
+      var pcaDs = data.Dataset.ToArray(data.AllowedInputVariables, data.TrainingIndices);
 
       int info;

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs

@@ -99 +99 @@
       }
 
-      var data = AlglibUtil.PrepareInputMatrix(problemData.Dataset, problemData.AllowedInputVariables,
-                                               problemData.TrainingIndices);
+      var data = problemData.Dataset.ToArray(problemData.AllowedInputVariables,
+                                             problemData.TrainingIndices);
       var classes = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
 

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs

@@ -86 +86 @@
 
     public double[,] Reduce(IDataset dataset, IEnumerable<int> rows) {
-      var data = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      var data = dataset.ToArray(allowedInputVariables, rows);
 
       var targets = dataset.GetDoubleValues(TargetVariable, rows).ToArray();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs

@@ -22 +22 @@
 using System;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -91 +92 @@
 
     #region nearest neighbour
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs

@@ -119 +119 @@
       if (IsCompatibilityLoaded) {
         // no scaling
-        inputMatrix = AlglibUtil.PrepareInputMatrix(dataset,
+        inputMatrix = dataset.ToArray(
           this.allowedInputVariables.Concat(new string[] { targetVariable }),
           rows);
@@ -144 +144 @@
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException(
-          "Nearest neighbour classification does not support NaN or infinity values in the input dataset.");
+          "Nearest neighbour model does not support NaN or infinity values in the input dataset.");
 
       this.kdTree = new alglib.nearestneighbor.kdtree();
@@ -167 +167 @@
 
     private static double[,] CreateScaledData(IDataset dataset, IEnumerable<string> variables, IEnumerable<int> rows, double[] offsets, double[] factors) {
-      var x = new double[rows.Count(), variables.Count()];
-      var colIdx = 0;
-      foreach (var variableName in variables) {
-        var rowIdx = 0;
-        foreach (var val in dataset.GetDoubleValues(variableName, rows)) {
-          x[rowIdx, colIdx] = (val + offsets[colIdx]) * factors[colIdx];
-          rowIdx++;
-        }
-        colIdx++;
-      }
-      return x;
+      var transforms =
+        variables.Select(
+          (_, colIdx) =>
+            new LinearTransformation(variables) { Addend = offsets[colIdx] * factors[colIdx], Multiplier = factors[colIdx] });
+      return dataset.ToArray(variables, transforms, rows);
     }
 
@@ -187 +181 @@
       double[,] inputData;
       if (IsCompatibilityLoaded) {
-        inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+        inputData = dataset.ToArray(allowedInputVariables, rows);
       } else {
         inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);
@@ -223 +217 @@
       double[,] inputData;
       if (IsCompatibilityLoaded) {
-        inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+        inputData = dataset.ToArray(allowedInputVariables, rows);
       } else {
         inputData = CreateScaledData(dataset, allowedInputVariables, rows, offsets, weights);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs

@@ -21 +21 @@
 
 using System;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -92 +93 @@
 
     #region nearest neighbour
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double[] weights = null;
       if (Weights != null) weights = Weights.CloneAsArray();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -168 +169 @@
 
     #region neural network
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, avgRelError, relClassError;
       var solution = CreateNeuralNetworkClassificationSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
@@ -183 +184 @@
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Neural network classification does not support NaN or infinity values in the input dataset.");

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -124 +125 @@
     public NeuralNetworkEnsembleClassification()
       : base() {
-      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
-        (IntValue)new IntValue(0).AsReadOnly(), 
-        (IntValue)new IntValue(1).AsReadOnly(), 
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] {
+        (IntValue)new IntValue(0).AsReadOnly(),
+        (IntValue)new IntValue(1).AsReadOnly(),
         (IntValue)new IntValue(2).AsReadOnly() });
       var selectedHiddenLayerValue = (from v in validHiddenLayerValues
@@ -154 +155 @@
 
     #region neural network ensemble
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, avgRelError, relClassError;
       var solution = CreateNeuralNetworkEnsembleClassificationSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
@@ -169 +170 @@
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Neural network ensemble classification does not support NaN or infinity values in the input dataset.");

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs

@@ -91 +91 @@
 
     public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
 
       int n = inputData.GetLength(0);
@@ -108 +108 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
 
       int n = inputData.GetLength(0);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -154 +155 @@
 
     #region neural network ensemble
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, avgRelError;
       var solution = CreateNeuralNetworkEnsembleRegressionSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
@@ -168 +169 @@
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Neural network ensemble regression does not support NaN or infinity values in the input dataset.");

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs

@@ -95 +95 @@
 
     public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(allowedInputVariables, rows);
 
       int n = inputData.GetLength(0);
@@ -112 +112 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray( allowedInputVariables, rows);
 
       int n = inputData.GetLength(0);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -170 +171 @@
 
     #region neural network
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, avgRelError;
       var solution = CreateNeuralNetworkRegressionSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
@@ -184 +185 @@
       IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables.Concat(new string[] { targetVariable }), rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Neural network regression does not support NaN or infinity values in the input dataset.");

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Analysis;
 using HeuristicLab.Common;
@@ -157 +159 @@
 
     #region nonlinear regression
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       IRegressionSolution bestSolution = null;
       if (InitializeParametersRandomly) {
@@ -207 +209 @@
       var parser = new InfixExpressionParser();
       var tree = parser.Parse(modelStructure);
+      // parser handles double and string variables equally by creating a VariableTreeNode
+      // post-process to replace VariableTreeNodes by FactorVariableTreeNodes for all string variables
+      var factorSymbol = new FactorVariable();
+      factorSymbol.VariableNames =
+        problemData.AllowedInputVariables.Where(name => problemData.Dataset.VariableHasType<string>(name));
+      factorSymbol.AllVariableNames = factorSymbol.VariableNames;
+      factorSymbol.VariableValues =
+        factorSymbol.VariableNames.Select(name =>
+        new KeyValuePair<string, Dictionary<string, int>>(name,
+        problemData.Dataset.GetReadOnlyStringValues(name).Distinct()
+        .Select((n, i) => Tuple.Create(n, i))
+        .ToDictionary(tup => tup.Item1, tup => tup.Item2)));
+
+      foreach (var parent in tree.IterateNodesPrefix().ToArray()) {
+        for (int i = 0; i < parent.SubtreeCount; i++) {
+          var varChild = parent.GetSubtree(i) as VariableTreeNode;
+          var factorVarChild = parent.GetSubtree(i) as FactorVariableTreeNode;
+          if (varChild != null && factorSymbol.VariableNames.Contains(varChild.VariableName)) {
+            parent.RemoveSubtree(i);
+            var factorTreeNode = (FactorVariableTreeNode)factorSymbol.CreateTreeNode();
+            factorTreeNode.VariableName = varChild.VariableName;
+            factorTreeNode.Weights =
+              factorTreeNode.Symbol.GetVariableValues(factorTreeNode.VariableName).Select(_ => 1.0).ToArray();
+            // weight = 1.0 for each value
+            parent.InsertSubtree(i, factorTreeNode);
+          } else if (factorVarChild != null && factorSymbol.VariableNames.Contains(factorVarChild.VariableName)) {
+            if (factorSymbol.GetVariableValues(factorVarChild.VariableName).Count() != factorVarChild.Weights.Length)
+              throw new ArgumentException(
+                string.Format("Factor variable {0} needs exactly {1} weights",
+                factorVarChild.VariableName,
+                factorSymbol.GetVariableValues(factorVarChild.VariableName).Count()));
+            parent.RemoveSubtree(i);
+            var factorTreeNode = (FactorVariableTreeNode)factorSymbol.CreateTreeNode();
+            factorTreeNode.VariableName = factorVarChild.VariableName;
+            factorTreeNode.Weights = factorVarChild.Weights;
+            parent.InsertSubtree(i, factorTreeNode);
+          }
+        }
+      }
 
       if (!SymbolicRegressionConstantOptimizationEvaluator.CanOptimizeConstants(tree)) throw new ArgumentException("The optimizer does not support the specified model structure.");

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs

@@ -20 +20 @@
 #endregion
 
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -132 +133 @@
 
     #region random forest
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, relClassificationError, outOfBagRmsError, outOfBagRelClassificationError;
       if (SetSeedRandomly) Seed = new System.Random().Next();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs

@@ -139 +139 @@
 
     public IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
       AssertInputMatrix(inputData);
 
@@ -157 +157 @@
 
     public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
       AssertInputMatrix(inputData);
 
@@ -175 +175 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
-      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, AllowedInputVariables, rows);
+      double[,] inputData = dataset.ToArray(AllowedInputVariables, rows);
       AssertInputMatrix(inputData);
 
@@ -294 +294 @@
       out double rmsError, out double outOfBagRmsError, out double avgRelError, out double outOfBagAvgRelError) {
       var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+      double[,] inputMatrix = problemData.Dataset.ToArray(variables, trainingIndices);
 
       alglib.dfreport rep;
@@ -316 +316 @@
 
       var variables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(problemData.Dataset, variables, trainingIndices);
+      double[,] inputMatrix = problemData.Dataset.ToArray(variables, trainingIndices);
 
       var classValues = problemData.ClassValues.ToArray();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs

@@ -20 +20 @@
 #endregion
 
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -131 +132 @@
 
     #region random forest
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, avgRelError, outOfBagRmsError, outOfBagAvgRelError;
       if (SetSeedRandomly) Seed = new System.Random().Next();

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -143 +144 @@
 
     #region support vector classification
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       IClassificationProblemData problemData = Problem.ProblemData;
       IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -151 +152 @@
 
     #region support vector regression
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       IRegressionProblemData problemData = Problem.ProblemData;
       IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs

@@ -22 +22 @@
 using System;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -63 +64 @@
     }
 
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       double rmsError, cvRmsError;
       var solution = CreateAutoRegressiveSolution(Problem.ProblemData, TimeOffset, out rmsError, out cvRmsError);
@@ -114 +115 @@
       alglib.lrunpack(lm, out coefficients, out nFeatures);
 
-
-      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
-      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
-      tree.Root.AddSubtree(startNode);
-      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
-      startNode.AddSubtree(addition);
-
-      for (int i = 0; i < timeOffset; i++) {
-        LaggedVariableTreeNode node = (LaggedVariableTreeNode)new LaggedVariable().CreateTreeNode();
-        node.VariableName = targetVariable;
-        node.Weight = coefficients[i];
-        node.Lag = (i + 1) * -1;
-        addition.AddSubtree(node);
-      }
-
-      ConstantTreeNode cNode = (ConstantTreeNode)new Constant().CreateTreeNode();
-      cNode.Value = coefficients[coefficients.Length - 1];
-      addition.AddSubtree(cNode);
+      var tree = LinearModelToTreeConverter.CreateTree(
+        variableNames: Enumerable.Repeat(problemData.TargetVariable, nFeatures).ToArray(),
+        lags: Enumerable.Range(0, timeOffset).Select(i => (i + 1) * -1).ToArray(),
+        coefficients: coefficients.Take(nFeatures).ToArray(),
+        @const: coefficients[nFeatures]
+        );
 
       var interpreter = new SymbolicTimeSeriesPrognosisExpressionTreeInterpreter(problemData.TargetVariable);

branches/RBFRegression/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -77 +78 @@
 
     #region k-Means clustering
-    protected override void Run() {
+    protected override void Run(CancellationToken cancellationToken) {
       var solution = CreateKMeansSolution(Problem.ProblemData, K.Value, Restarts.Value);
       Results.Add(new Result(KMeansSolutionResultName, "The k-Means clustering solution.", solution));
@@ -89 +90 @@
       double[,] centers;
       int[] xyc;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] inputMatrix = dataset.ToArray(allowedInputVariables, rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("k-Means clustering does not support NaN or infinity values in the input dataset.");