Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs	(revision 9070)
@@ -0,0 +1,756 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Collections;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
+    public CrossValidation()
+      : base() {
+      name = ItemName;
+      description = ItemDescription;
+
+      executionState = ExecutionState.Stopped;
+      runs = new RunCollection { OptimizerName = name };
+      runsCounter = 0;
+
+      algorithm = null;
+      clonedAlgorithms = new ItemCollection<IAlgorithm>();
+      results = new ResultCollection();
+
+      folds = new IntValue(2);
+      numberOfWorkers = new IntValue(1);
+      samplesStart = new IntValue(0);
+      samplesEnd = new IntValue(0);
+      storeAlgorithmInEachRun = false;
+
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+
+    public string Filename { get; set; }
+
+    #region persistence and cloning
+    [StorableConstructor]
+    private CrossValidation(bool deserializing)
+      : base(deserializing) {
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+
+    private CrossValidation(CrossValidation original, Cloner cloner)
+      : base(original, cloner) {
+      executionState = original.executionState;
+      storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
+      runs = cloner.Clone(original.runs);
+      runsCounter = original.runsCounter;
+      algorithm = cloner.Clone(original.algorithm);
+      clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
+      results = cloner.Clone(original.results);
+
+      folds = cloner.Clone(original.folds);
+      numberOfWorkers = cloner.Clone(original.numberOfWorkers);
+      samplesStart = cloner.Clone(original.samplesStart);
+      samplesEnd = cloner.Clone(original.samplesEnd);
+      RegisterEvents();
+      if (Algorithm != null) RegisterAlgorithmEvents();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CrossValidation(this, cloner);
+    }
+
+    #endregion
+
+    #region properties
+    [Storable]
+    private IAlgorithm algorithm;
+    public IAlgorithm Algorithm {
+      get { return algorithm; }
+      set {
+        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+          throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
+        if (algorithm != value) {
+          if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
+            throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
+          if (algorithm != null) DeregisterAlgorithmEvents();
+          algorithm = value;
+          Parameters.Clear();
+
+          if (algorithm != null) {
+            algorithm.StoreAlgorithmInEachRun = false;
+            RegisterAlgorithmEvents();
+            algorithm.Prepare(true);
+            Parameters.AddRange(algorithm.Parameters);
+          }
+          OnAlgorithmChanged();
+          Prepare();
+        }
+      }
+    }
+
+
+    [Storable]
+    private IDataAnalysisProblem problem;
+    public IDataAnalysisProblem Problem {
+      get {
+        if (algorithm == null)
+          return null;
+        return (IDataAnalysisProblem)algorithm.Problem;
+      }
+      set {
+        if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+          throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
+        if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
+        algorithm.Problem = value;
+        problem = value;
+      }
+    }
+
+    IProblem IAlgorithm.Problem {
+      get { return Problem; }
+      set {
+        if (value != null && !ProblemType.IsInstanceOfType(value))
+          throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
+        Problem = (IDataAnalysisProblem)value;
+      }
+    }
+    public Type ProblemType {
+      get { return typeof(IDataAnalysisProblem); }
+    }
+
+    [Storable]
+    private ItemCollection<IAlgorithm> clonedAlgorithms;
+
+    public IEnumerable<IOptimizer> NestedOptimizers {
+      get {
+        if (Algorithm == null) yield break;
+        yield return Algorithm;
+      }
+    }
+
+    [Storable]
+    private ResultCollection results;
+    public ResultCollection Results {
+      get { return results; }
+    }
+
+    [Storable]
+    private IntValue folds;
+    public IntValue Folds {
+      get { return folds; }
+    }
+    [Storable]
+    private IntValue samplesStart;
+    public IntValue SamplesStart {
+      get { return samplesStart; }
+    }
+    [Storable]
+    private IntValue samplesEnd;
+    public IntValue SamplesEnd {
+      get { return samplesEnd; }
+    }
+    [Storable]
+    private IntValue numberOfWorkers;
+    public IntValue NumberOfWorkers {
+      get { return numberOfWorkers; }
+    }
+
+    [Storable]
+    private bool storeAlgorithmInEachRun;
+    public bool StoreAlgorithmInEachRun {
+      get { return storeAlgorithmInEachRun; }
+      set {
+        if (storeAlgorithmInEachRun != value) {
+          storeAlgorithmInEachRun = value;
+          OnStoreAlgorithmInEachRunChanged();
+        }
+      }
+    }
+
+    [Storable]
+    private int runsCounter;
+    [Storable]
+    private RunCollection runs;
+    public RunCollection Runs {
+      get { return runs; }
+    }
+
+    [Storable]
+    private ExecutionState executionState;
+    public ExecutionState ExecutionState {
+      get { return executionState; }
+      private set {
+        if (executionState != value) {
+          executionState = value;
+          OnExecutionStateChanged();
+          OnItemImageChanged();
+        }
+      }
+    }
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
+    }
+    public override Image ItemImage {
+      get {
+        if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
+        else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
+        else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
+        else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
+        else return base.ItemImage;
+      }
+    }
+
+    public TimeSpan ExecutionTime {
+      get {
+        if (ExecutionState != ExecutionState.Prepared)
+          return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
+        return TimeSpan.Zero;
+      }
+    }
+    #endregion
+
+    protected override void OnNameChanged() {
+      base.OnNameChanged();
+      Runs.OptimizerName = Name;
+    }
+
+    public void Prepare() {
+      if (ExecutionState == ExecutionState.Started)
+        throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
+      results.Clear();
+      clonedAlgorithms.Clear();
+      if (Algorithm != null) {
+        Algorithm.Prepare();
+        if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
+      }
+    }
+    public void Prepare(bool clearRuns) {
+      if (clearRuns) runs.Clear();
+      Prepare();
+    }
+
+    private bool startPending;
+    public void Start() {
+      if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
+        throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
+
+      if (Algorithm != null && !startPending) {
+        startPending = true;
+        //create cloned algorithms
+        if (clonedAlgorithms.Count == 0) {
+          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
+
+          for (int i = 0; i < Folds.Value; i++) {
+            IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
+            clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
+            IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
+            ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
+
+            int testStart = (i * testSamplesCount) + SamplesStart.Value;
+            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
+
+            problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
+            problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
+            problem.ProblemData.TestPartition.Start = testStart;
+            problem.ProblemData.TestPartition.End = testEnd;
+            DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
+            if (problemData != null) {
+              problemData.TrainingPartitionParameter.Hidden = false;
+              problemData.TestPartitionParameter.Hidden = false;
+            }
+
+            if (symbolicProblem != null) {
+              symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+              symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+            }
+            clonedAlgorithm.Prepare();
+            clonedAlgorithms.Add(clonedAlgorithm);
+          }
+        }
+
+        //start prepared or paused cloned algorithms
+        int startedAlgorithms = 0;
+        foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
+          if (startedAlgorithms < NumberOfWorkers.Value) {
+            if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
+                clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
+
+              // start and wait until the alg is started 
+              using (var signal = new ManualResetEvent(false)) {
+                EventHandler signalSetter = (sender, args) => { signal.Set(); };
+                clonedAlgorithm.Started += signalSetter;
+                clonedAlgorithm.Start();
+                signal.WaitOne();
+                clonedAlgorithm.Started -= signalSetter;
+
+                startedAlgorithms++;
+              }
+            }
+          }
+        }
+        OnStarted();
+      }
+    }
+
+    private bool pausePending;
+    public void Pause() {
+      if (ExecutionState != ExecutionState.Started)
+        throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
+      if (!pausePending) {
+        pausePending = true;
+        if (!startPending) PauseAllClonedAlgorithms();
+      }
+    }
+    private void PauseAllClonedAlgorithms() {
+      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
+        if (clonedAlgorithm.ExecutionState == ExecutionState.Started)
+          clonedAlgorithm.Pause();
+      }
+    }
+
+    private bool stopPending;
+    public void Stop() {
+      if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
+        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
+                                                          ExecutionState));
+      if (!stopPending) {
+        stopPending = true;
+        if (!startPending) StopAllClonedAlgorithms();
+      }
+    }
+    private void StopAllClonedAlgorithms() {
+      foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
+        if (clonedAlgorithm.ExecutionState == ExecutionState.Started ||
+            clonedAlgorithm.ExecutionState == ExecutionState.Paused)
+          clonedAlgorithm.Stop();
+      }
+    }
+
+    #region collect parameters and results
+    public override void CollectParameterValues(IDictionary<string, IItem> values) {
+      values.Add("Algorithm Name", new StringValue(Name));
+      values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
+      values.Add("Folds", new IntValue(Folds.Value));
+
+      if (algorithm != null) {
+        values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
+        values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
+        base.CollectParameterValues(values);
+      }
+      if (Problem != null) {
+        values.Add("Problem Name", new StringValue(Problem.Name));
+        values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
+        Problem.CollectParameterValues(values);
+      }
+    }
+
+    public void CollectResultValues(IDictionary<string, IItem> results) {
+      var clonedResults = (ResultCollection)this.results.Clone();
+      foreach (var result in clonedResults) {
+        results.Add(result.Name, result.Value);
+      }
+    }
+
+    private void AggregateResultValues(IDictionary<string, IItem> results) {
+      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
+      IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
+      IEnumerable<KeyValuePair<string, IItem>> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
+
+      foreach (IResult result in ExtractAndAggregateResults<IntValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateResults<DoubleValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateResults<PercentValue>(resultCollections))
+        results.Add(result.Name, result.Value);
+      foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
+        results.Add(result.Name, result.Value);
+      }
+      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
+        results.Add(result.Name, result.Value);
+      }
+      results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
+      results.Add("CrossValidation Folds", new RunCollection(runs));
+    }
+
+    private IEnumerable<IResult> ExtractAndAggregateRegressionSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
+      Dictionary<string, List<IRegressionSolution>> resultSolutions = new Dictionary<string, List<IRegressionSolution>>();
+      foreach (var result in resultCollections) {
+        var regressionSolution = result.Value as IRegressionSolution;
+        if (regressionSolution != null) {
+          if (resultSolutions.ContainsKey(result.Key)) {
+            resultSolutions[result.Key].Add(regressionSolution);
+          } else {
+            resultSolutions.Add(result.Key, new List<IRegressionSolution>() { regressionSolution });
+          }
+        }
+      }
+      List<IResult> aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
+        // clone manually to correctly clone references between cloned root objects 
+        Cloner cloner = new Cloner();
+        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
+        // set partitions of problem data clone correctly
+        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+        // clone models
+        var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
+        ensembleSolution.AddRegressionSolutions(solutions.Value);
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
+      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
+      foreach (var result in resultCollections) {
+        var classificationSolution = result.Value as IClassificationSolution;
+        if (classificationSolution != null) {
+          if (resultSolutions.ContainsKey(result.Key)) {
+            resultSolutions[result.Key].Add(classificationSolution);
+          } else {
+            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
+          }
+        }
+      }
+      var aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
+        // clone manually to correctly clone references between cloned root objects 
+        Cloner cloner = new Cloner();
+        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
+        // set partitions of problem data clone correctly
+        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+        // clone models
+        var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
+        ensembleSolution.AddClassificationSolutions(solutions.Value);
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
+      foreach (IResult result in results) {
+        flattenedResults.Add(new Result(path + result.Name, result.Value));
+        ResultCollection childCollection = result.Value as ResultCollection;
+        if (childCollection != null) {
+          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
+        }
+      }
+    }
+
+    private static IEnumerable<IResult> ExtractAndAggregateResults<T>(IEnumerable<KeyValuePair<string, IItem>> results)
+  where T : class, IItem, new() {
+      Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
+      foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
+        if (!resultValues.ContainsKey(resultValue.Key))
+          resultValues[resultValue.Key] = new List<double>();
+        resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
+      }
+
+      DoubleValue doubleValue;
+      if (typeof(T) == typeof(PercentValue))
+        doubleValue = new PercentValue();
+      else if (typeof(T) == typeof(DoubleValue))
+        doubleValue = new DoubleValue();
+      else if (typeof(T) == typeof(IntValue))
+        doubleValue = new DoubleValue();
+      else
+        throw new NotSupportedException();
+
+      List<IResult> aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
+        doubleValue.Value = resultValue.Value.Average();
+        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
+        doubleValue.Value = resultValue.Value.StandardDeviation();
+        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
+      }
+      return aggregatedResults;
+    }
+
+    private static double ConvertToDouble(IItem item) {
+      if (item is DoubleValue) return ((DoubleValue)item).Value;
+      else if (item is IntValue) return ((IntValue)item).Value;
+      else throw new NotSupportedException("Could not convert any item type to double");
+    }
+    #endregion
+
+    #region events
+    private void RegisterEvents() {
+      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
+      RegisterClonedAlgorithmsEvents();
+    }
+    private void Folds_ValueChanged(object sender, EventArgs e) {
+      if (ExecutionState != ExecutionState.Prepared)
+        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
+    }
+
+
+    #region template algorithms events
+    public event EventHandler AlgorithmChanged;
+    private void OnAlgorithmChanged() {
+      EventHandler handler = AlgorithmChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+      OnProblemChanged();
+      if (Problem == null) ExecutionState = ExecutionState.Stopped;
+    }
+    private void RegisterAlgorithmEvents() {
+      algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
+      algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
+      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
+    }
+    private void DeregisterAlgorithmEvents() {
+      algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
+      algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
+      if (Problem != null) Problem.Reset -= new EventHandler(Problem_Reset);
+    }
+    private void Algorithm_ProblemChanged(object sender, EventArgs e) {
+      if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
+        algorithm.Problem = problem;
+        throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
+      }
+      if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
+      if (Problem != null) Problem.Reset += new EventHandler(Problem_Reset);
+      problem = (IDataAnalysisProblem)algorithm.Problem;
+      OnProblemChanged();
+    }
+    public event EventHandler ProblemChanged;
+    private void OnProblemChanged() {
+      EventHandler handler = ProblemChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+      ConfigureProblem();
+    }
+
+    private void Problem_Reset(object sender, EventArgs e) {
+      ConfigureProblem();
+    }
+
+    private void ConfigureProblem() {
+      SamplesStart.Value = 0;
+      if (Problem != null) {
+        SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
+
+        DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
+        if (problemData != null) {
+          problemData.TrainingPartitionParameter.Hidden = true;
+          problemData.TestPartitionParameter.Hidden = true;
+        }
+        ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
+        if (symbolicProblem != null) {
+          symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
+          symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+          symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+          symbolicProblem.ValidationPartitionParameter.Hidden = true;
+          symbolicProblem.ValidationPartition.Start = 0;
+          symbolicProblem.ValidationPartition.End = 0;
+        }
+      } else
+        SamplesEnd.Value = 0;
+    }
+
+    private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
+      switch (Algorithm.ExecutionState) {
+        case ExecutionState.Prepared: OnPrepared();
+          break;
+        case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
+        case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
+        case ExecutionState.Stopped: OnStopped();
+          break;
+      }
+    }
+    #endregion
+
+    #region clonedAlgorithms events
+    private void RegisterClonedAlgorithmsEvents() {
+      clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
+      clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
+      clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
+      foreach (IAlgorithm algorithm in clonedAlgorithms)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void DeregisterClonedAlgorithmsEvents() {
+      clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsAdded);
+      clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_ItemsRemoved);
+      clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler<IAlgorithm>(ClonedAlgorithms_CollectionReset);
+      foreach (IAlgorithm algorithm in clonedAlgorithms)
+        DeregisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.Items)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.Items)
+        DeregisterClonedAlgorithmEvents(algorithm);
+    }
+    private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs<IAlgorithm> e) {
+      foreach (IAlgorithm algorithm in e.OldItems)
+        DeregisterClonedAlgorithmEvents(algorithm);
+      foreach (IAlgorithm algorithm in e.Items)
+        RegisterClonedAlgorithmEvents(algorithm);
+    }
+    private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+      algorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
+      algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+      algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
+      algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
+      algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
+    }
+    private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+      algorithm.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(ClonedAlgorithm_ExceptionOccurred);
+      algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+      algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
+      algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
+      algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
+    }
+    private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
+      OnExceptionOccurred(e.Value);
+    }
+    private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
+      OnExecutionTimeChanged();
+    }
+
+    private readonly object locker = new object();
+    private void ClonedAlgorithm_Started(object sender, EventArgs e) {
+      lock (locker) {
+        IAlgorithm algorithm = sender as IAlgorithm;
+        if (algorithm != null && !results.ContainsKey(algorithm.Name))
+          results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
+
+        if (startPending) {
+          int startedAlgorithms = clonedAlgorithms.Count(alg => alg.ExecutionState == ExecutionState.Started);
+          if (startedAlgorithms == NumberOfWorkers.Value ||
+             clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Prepared))
+            startPending = false;
+
+          if (pausePending) PauseAllClonedAlgorithms();
+          if (stopPending) StopAllClonedAlgorithms();
+        }
+      }
+    }
+
+    private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
+      lock (locker) {
+        if (pausePending && clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started))
+          OnPaused();
+      }
+    }
+
+    private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
+      lock (locker) {
+        if (!stopPending && ExecutionState == ExecutionState.Started) {
+          IAlgorithm preparedAlgorithm = clonedAlgorithms.FirstOrDefault(alg => alg.ExecutionState == ExecutionState.Prepared ||
+                                                                                alg.ExecutionState == ExecutionState.Paused);
+          if (preparedAlgorithm != null) preparedAlgorithm.Start();
+        }
+        if (ExecutionState != ExecutionState.Stopped) {
+          if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped))
+            OnStopped();
+          else if (stopPending &&
+                   clonedAlgorithms.All(
+                     alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped))
+            OnStopped();
+        }
+      }
+    }
+    #endregion
+    #endregion
+
+    #region event firing
+    public event EventHandler ExecutionStateChanged;
+    private void OnExecutionStateChanged() {
+      EventHandler handler = ExecutionStateChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler ExecutionTimeChanged;
+    private void OnExecutionTimeChanged() {
+      EventHandler handler = ExecutionTimeChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Prepared;
+    private void OnPrepared() {
+      ExecutionState = ExecutionState.Prepared;
+      EventHandler handler = Prepared;
+      if (handler != null) handler(this, EventArgs.Empty);
+      OnExecutionTimeChanged();
+    }
+    public event EventHandler Started;
+    private void OnStarted() {
+      startPending = false;
+      ExecutionState = ExecutionState.Started;
+      EventHandler handler = Started;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Paused;
+    private void OnPaused() {
+      pausePending = false;
+      ExecutionState = ExecutionState.Paused;
+      EventHandler handler = Paused;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler Stopped;
+    private void OnStopped() {
+      stopPending = false;
+      Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
+      AggregateResultValues(collectedResults);
+      results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
+      runsCounter++;
+      runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
+      ExecutionState = ExecutionState.Stopped;
+      EventHandler handler = Stopped;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;
+    private void OnExceptionOccurred(Exception exception) {
+      EventHandler<EventArgs<Exception>> handler = ExceptionOccurred;
+      if (handler != null) handler(this, new EventArgs<Exception>(exception));
+    }
+    public event EventHandler StoreAlgorithmInEachRunChanged;
+    private void OnStoreAlgorithmInEachRunChanged() {
+      EventHandler handler = StoreAlgorithmInEachRunChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs	(revision 9070)
@@ -0,0 +1,131 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  public abstract class FixedDataAnalysisAlgorithm<T> : Algorithm,
+    IDataAnalysisAlgorithm<T>,
+    IStorableContent
+    where T : class, IDataAnalysisProblem {
+    public string Filename { get; set; }
+
+    #region Properties
+    public override Type ProblemType {
+      get { return typeof(T); }
+    }
+    public new T Problem {
+      get { return (T)base.Problem; }
+      set { base.Problem = value; }
+    }
+    [Storable]
+    private ResultCollection results;
+    public override ResultCollection Results {
+      get { return results; }
+    }
+    #endregion
+
+    private DateTime lastUpdateTime;
+
+    [StorableConstructor]
+    protected FixedDataAnalysisAlgorithm(bool deserializing) : base(deserializing) { }
+    protected FixedDataAnalysisAlgorithm(FixedDataAnalysisAlgorithm<T> original, Cloner cloner)
+      : base(original, cloner) {
+      results = cloner.Clone(original.Results);
+    }
+    public FixedDataAnalysisAlgorithm()
+      : base() {
+      results = new ResultCollection();
+    }
+
+    public override void Prepare() {
+      if (Problem != null) base.Prepare();
+      results.Clear();
+      OnPrepared();
+    }
+
+    public override void Start() {
+      base.Start();
+      var cancellationTokenSource = new CancellationTokenSource();
+
+      OnStarted();
+      Task task = Task.Factory.StartNew(Run, cancellationTokenSource.Token, cancellationTokenSource.Token);
+      task.ContinueWith(t => {
+        try {
+          t.Wait();
+        }
+        catch (AggregateException ex) {
+          try {
+            ex.Flatten().Handle(x => x is OperationCanceledException);
+          }
+          catch (AggregateException remaining) {
+            if (remaining.InnerExceptions.Count == 1) OnExceptionOccurred(remaining.InnerExceptions[0]);
+            else OnExceptionOccurred(remaining);
+          }
+        }
+        cancellationTokenSource.Dispose();
+        cancellationTokenSource = null;
+        OnStopped();
+      });
+    }
+    private void Run(object state) {
+      CancellationToken cancellationToken = (CancellationToken)state;
+      lastUpdateTime = DateTime.Now;
+      System.Timers.Timer timer = new System.Timers.Timer(250);
+      timer.AutoReset = true;
+      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
+      timer.Start();
+      try {
+        Run();
+      }
+      finally {
+        timer.Elapsed -= new System.Timers.ElapsedEventHandler(timer_Elapsed);
+        timer.Stop();
+        ExecutionTime += DateTime.Now - lastUpdateTime;
+      }
+
+      cancellationToken.ThrowIfCancellationRequested();
+    }
+    protected abstract void Run();
+    #region Events
+    protected override void OnProblemChanged() {
+      Problem.Reset += new EventHandler(Problem_Reset);
+      base.OnProblemChanged();
+    }
+    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
+      System.Timers.Timer timer = (System.Timers.Timer)sender;
+      timer.Enabled = false;
+      DateTime now = DateTime.Now;
+      ExecutionTime += now - lastUpdateTime;
+      lastUpdateTime = now;
+      timer.Enabled = true;
+    }
+    #endregion
+
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs	(revision 9070)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceConst",
+    Description = "Constant covariance function for Gaussian processes.")]
+  public sealed class CovarianceConst : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceConst(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceConst(CovarianceConst original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceConst()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of the constant covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceConst(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return ScaleParameter.Value != null ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      int c = 0;
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceConst", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => scale;
+      cov.CrossCovariance = (x, xt, i, j) => scale;
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, scale, columnIndices);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double scale, IEnumerable<int> columnIndices) {
+      yield return 2.0 * scale;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs	(revision 9070)
@@ -0,0 +1,64 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceLinear", Description = "Linear covariance function for Gaussian processes.")]
+  public sealed class CovarianceLinear : Item, ICovarianceFunction {
+    [StorableConstructor]
+    private CovarianceLinear(bool deserializing) : base(deserializing) { }
+    private CovarianceLinear(CovarianceLinear original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceLinear()
+      : base() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceLinear(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, 1, columnIndices);
+      cov.CrossCovariance = (x, xt, i, j) =>  Util.ScalarProd(x, i, xt, j, 1.0 , columnIndices);
+      cov.CovarianceGradient = (x, i, j) => Enumerable.Empty<double>();
+      return cov;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs	(revision 9070)
@@ -0,0 +1,101 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceLinearArd",
+    Description = "Linear covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceLinearArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceLinearArd(bool deserializing) : base(deserializing) { }
+    private CovarianceLinearArd(CovarianceLinearArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceLinearArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength",
+                                                             "The inverse length parameter for ARD."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceLinearArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      if (InverseLengthParameter.Value == null)
+        return numberOfVariables;
+      else
+        return 0;
+    }
+
+    public void SetParameter(double[] p) {
+      double[] inverseLength;
+      GetParameterValues(p, out inverseLength);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double[] inverseLength) {
+      // gather parameter values
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        inverseLength = p.Select(e => 1.0 / Math.Exp(e)).ToArray();
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double[] inverseLength;
+      GetParameterValues(p, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, inverseLength, columnIndices);
+      cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, inverseLength, columnIndices);
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, inverseLength, columnIndices);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double[] inverseLength, IEnumerable<int> columnIndices) {
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+
+      int k = 0;
+      foreach (int columnIndex in columnIndices) {
+        yield return -2.0 * x[i, columnIndex] * x[j, columnIndex] * inverseLength[k] * inverseLength[k];
+        k++;
+      }
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs	(revision 9070)
@@ -0,0 +1,84 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceMask",
+    Description = "Masking covariance function for dimension selection can be used to apply a covariance function only on certain input dimensions.")]
+  public sealed class CovarianceMask : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<IntArray> SelectedDimensionsParameter {
+      get { return (IValueParameter<IntArray>)Parameters["SelectedDimensions"]; }
+    }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters["CovarianceFunction"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceMask(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceMask(CovarianceMask original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceMask()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<IntArray>("SelectedDimensions", "The dimensions on which the specified covariance function should be applied to."));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceMask(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      if (SelectedDimensionsParameter.Value == null) return CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+      else return CovarianceFunctionParameter.Value.GetNumberOfParameters(SelectedDimensionsParameter.Value.Length);
+    }
+
+    public void SetParameter(double[] p) {
+      CovarianceFunctionParameter.Value.SetParameter(p);
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      if (columnIndices != null)
+        throw new InvalidOperationException("Stacking of masking covariance functions is not supported.");
+      var cov = CovarianceFunctionParameter.Value;
+      var selectedDimensions = SelectedDimensionsParameter.Value;
+
+      return cov.GetParameterizedCovarianceFunction(p, selectedDimensions);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs	(revision 9070)
@@ -0,0 +1,160 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceMaternIso",
+    Description = "Matern covariance function for Gaussian processes.")]
+  public sealed class CovarianceMaternIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IConstrainedValueParameter<IntValue> DParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters["D"]; }
+    }
+
+
+    [StorableConstructor]
+    private CovarianceMaternIso(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceMaternIso(CovarianceMaternIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceMaternIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric Matern covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric Matern covariance function."));
+      var validDValues = new ItemSet<IntValue>();
+      validDValues.Add((IntValue)new IntValue(1).AsReadOnly());
+      validDValues.Add((IntValue)new IntValue(3).AsReadOnly());
+      validDValues.Add((IntValue)new IntValue(5).AsReadOnly());
+      Parameters.Add(new ConstrainedValueParameter<IntValue>("D", "The d parameter (allowed values: 1, 3, or 5) of the isometric Matern covariance function.", validDValues, validDValues.First()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceMaternIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (InverseLengthParameter.Value != null ? 0 : 1) +
+        (ScaleParameter.Value != null ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double inverseLength, scale;
+      GetParameterValues(p, out scale, out inverseLength);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceMaternIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double inverseLength, scale;
+      int d = DParameter.Value.Value;
+      GetParameterValues(p, out scale, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double dist = i == j
+                       ? 0.0
+                       : Math.Sqrt(Util.SqrDist(x, i, j, Math.Sqrt(d) * inverseLength, columnIndices));
+        return scale * m(d, dist);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double dist = Math.Sqrt(Util.SqrDist(x, i, xt, j, Math.Sqrt(d) * inverseLength, columnIndices));
+        return scale * m(d, dist);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, d, scale, inverseLength, columnIndices);
+      return cov;
+    }
+
+    private static double m(int d, double t) {
+      double f;
+      switch (d) {
+        case 1: { f = 1; break; }
+        case 3: { f = 1 + t; break; }
+        case 5: { f = 1 + t * (1 + t / 3.0); break; }
+        default: throw new InvalidOperationException();
+      }
+      return f * Math.Exp(-t);
+    }
+
+    private static double dm(int d, double t) {
+      double df;
+      switch (d) {
+        case 1: { df = 1; break; }
+        case 3: { df = t; break; }
+        case 5: { df = t * (1 + t) / 3.0; break; }
+        default: throw new InvalidOperationException();
+      }
+      return df * t * Math.Exp(-t);
+    }
+
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, int d, double scale, double inverseLength, IEnumerable<int> columnIndices) {
+      double dist = i == j
+                   ? 0.0
+                   : Math.Sqrt(Util.SqrDist(x, i, j, Math.Sqrt(d) * inverseLength, columnIndices));
+
+      yield return scale * dm(d, dist);
+      yield return 2 * scale * m(d, dist);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs	(revision 9070)
@@ -0,0 +1,94 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceNoise",
+    Description = "Noise covariance function for Gaussian processes.")]
+  public sealed class CovarianceNoise : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceNoise(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceNoise(CovarianceNoise original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceNoise()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of noise."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceNoise(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return ScaleParameter.Value != null ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      int c = 0;
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNoise", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => i == j ? scale : 0.0;
+      cov.CrossCovariance = (x, xt, i, j) => 0.0;
+      cov.CovarianceGradient = (x, i, j) => Enumerable.Repeat(i == j ? 2.0 * scale : 0.0, 1);
+      return cov;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs	(revision 9070)
@@ -0,0 +1,148 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovariancePeriodic", Description = "Periodic covariance function for Gaussian processes.")]
+  public sealed class CovariancePeriodic : ParameterizedNamedItem, ICovarianceFunction {
+
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> PeriodParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Period"]; }
+    }
+
+
+    [StorableConstructor]
+    private CovariancePeriodic(bool deserializing) : base(deserializing) { }
+    private CovariancePeriodic(CovariancePeriodic original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovariancePeriodic()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale of the periodic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter for the periodic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Period", "The period parameter for the periodic covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovariancePeriodic(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (ScaleParameter.Value != null ? 0 : 1) +
+       (PeriodParameter.Value != null ? 0 : 1) +
+       (InverseLengthParameter.Value != null ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, inverseLength, period;
+      GetParameterValues(p, out scale, out period, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      PeriodParameter.Value = new DoubleValue(period);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+
+    private void GetParameterValues(double[] p, out double scale, out double period, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+      if (PeriodParameter.Value != null) {
+        period = PeriodParameter.Value.Value;
+      } else {
+        period = Math.Exp(p[c]);
+        c++;
+      }
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePeriodic", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double inverseLength, period, scale;
+      GetParameterValues(p, out scale, out period, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double k = i == j ? 0.0 : GetDistance(x, x, i, j, columnIndices);
+        k = Math.PI * k / period;
+        k = Math.Sin(k) * inverseLength;
+        k = k * k;
+
+        return scale * Math.Exp(-2.0 * k);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double k = GetDistance(x, xt, i, j, columnIndices);
+        k = Math.PI * k / period;
+        k = Math.Sin(k) * inverseLength;
+        k = k * k;
+
+        return scale * Math.Exp(-2.0 * k);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, period, inverseLength);
+      return cov;
+    }
+
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double period, double inverseLength) {
+      double v = i == j ? 0.0 : Math.PI * GetDistance(x, x, i, j, columnIndices) / period;
+      double gradient = Math.Sin(v) * inverseLength;
+      gradient *= gradient;
+      yield return 4.0 * scale * Math.Exp(-2.0 * gradient) * gradient;
+      double r = Math.Sin(v) * inverseLength;
+      yield return 4.0 * scale * inverseLength * Math.Exp(-2 * r * r) * r * Math.Cos(v) * v;
+      yield return 2.0 * scale * Math.Exp(-2 * gradient);
+    }
+
+    private static double GetDistance(double[,] x, double[,] xt, int i, int j, IEnumerable<int> columnIndices) {
+      return Math.Sqrt(Util.SqrDist(x, i, xt, j, 1, columnIndices));
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs	(revision 9070)
@@ -0,0 +1,107 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceProduct",
+    Description = "Product covariance function for Gaussian processes.")]
+  public sealed class CovarianceProduct : Item, ICovarianceFunction {
+    [Storable]
+    private ItemList<ICovarianceFunction> factors;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<ICovarianceFunction> Factors {
+      get { return factors; }
+    }
+
+    [StorableConstructor]
+    private CovarianceProduct(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceProduct(CovarianceProduct original, Cloner cloner)
+      : base(original, cloner) {
+      this.factors = cloner.Clone(original.factors);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public CovarianceProduct()
+      : base() {
+      this.factors = new ItemList<ICovarianceFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceProduct(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return factors.Select(f => f.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var f in factors) {
+        var numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+        f.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      if (factors.Count == 0) throw new ArgumentException("at least one factor is necessary for the product covariance function.");
+      var functions = new List<ParameterizedCovarianceFunction>();
+      foreach (var f in factors) {
+        int numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+        functions.Add(f.GetParameterizedCovarianceFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+      }
+
+
+      var product = new ParameterizedCovarianceFunction();
+      product.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Aggregate((a, b) => a * b);
+      product.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Aggregate((a, b) => a * b);
+      product.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, functions);
+      return product;
+    }
+
+    public static IEnumerable<double> GetGradient(double[,] x, int i, int j, List<ParameterizedCovarianceFunction> factorFunctions) {
+      var covariances = factorFunctions.Select(f => f.Covariance(x, i, j)).ToArray();
+      for (int ii = 0; ii < factorFunctions.Count; ii++) {
+        foreach (var g in factorFunctions[ii].CovarianceGradient(x, i, j)) {
+          double res = g;
+          for (int jj = 0; jj < covariances.Length; jj++)
+            if (ii != jj) res *= covariances[jj];
+          yield return res;
+        }
+      }
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs	(revision 9070)
@@ -0,0 +1,146 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceRationalQuadraticArd",
+    Description = "Rational quadratic covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceRationalQuadraticArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ShapeParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Shape"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceRationalQuadraticArd(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceRationalQuadraticArd(CovarianceRationalQuadraticArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceRationalQuadraticArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the rational quadratic covariance function with ARD."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength", "The inverse length parameter for automatic relevance determination."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Shape", "The shape parameter (alpha) of the rational quadratic covariance function with ARD."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceRationalQuadraticArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (ScaleParameter.Value != null ? 0 : 1) +
+        (ShapeParameter.Value != null ? 0 : 1) +
+        (InverseLengthParameter.Value != null ? 0 : numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, shape;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      ShapeParameter.Value = new DoubleValue(shape);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double shape, out double[] inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (ShapeParameter.Value != null) {
+        shape = ShapeParameter.Value.Value;
+      } else {
+        shape = Math.Exp(p[c]);
+        c++;
+      }
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        inverseLength = p.Skip(2).Select(e => 1.0 / Math.Exp(e)).ToArray();
+        c += inverseLength.Length;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticArd", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale, shape;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                    ? 0.0
+                    : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double shape, double[] inverseLength) {
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+      double b = 1 + 0.5 * d / shape;
+      int k = 0;
+      foreach (var columnIndex in columnIndices) {
+        yield return scale * Math.Pow(b, -shape - 1) * Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]);
+        k++;
+      }
+      yield return 2 * scale * Math.Pow(b, -shape);
+      yield return scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b));
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs	(revision 9070)
@@ -0,0 +1,137 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceRationalQuadraticIso",
+    Description = "Isotropic rational quadratic covariance function for Gaussian processes.")]
+  public sealed class CovarianceRationalQuadraticIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    public IValueParameter<DoubleValue> ShapeParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Shape"]; }
+    }
+    [StorableConstructor]
+    private CovarianceRationalQuadraticIso(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceRationalQuadraticIso(CovarianceRationalQuadraticIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceRationalQuadraticIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric rational quadratic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric rational quadratic covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Shape", "The shape parameter (alpha) of the isometric rational quadratic covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceRationalQuadraticIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (ScaleParameter.Value != null ? 0 : 1) +
+        (ShapeParameter.Value != null ? 0 : 1) +
+        (InverseLengthParameter.Value != null ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, shape, inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      ShapeParameter.Value = new DoubleValue(shape);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double shape, out double inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (ShapeParameter.Value != null) {
+        shape = ShapeParameter.Value.Value;
+      } else {
+        shape = Math.Exp(p[c]);
+        c++;
+      }
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale, shape, inverseLength;
+      GetParameterValues(p, out scale, out shape, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                    ? 0.0
+                    : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return shape * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double shape, double inverseLength) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+
+      double b = 1 + 0.5 * d / shape;
+      yield return scale * Math.Pow(b, -shape - 1) * d;
+      yield return 2 * scale * Math.Pow(b, -shape);
+      yield return scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b));
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs	(revision 9070)
@@ -0,0 +1,104 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceScale",
+    Description = "Scale covariance function for Gaussian processes.")]
+  public sealed class CovarianceScale : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters["CovarianceFunction"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceScale(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceScale(CovarianceScale original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceScale()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter."));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceScale(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return (ScaleParameter.Value != null ? 0 : 1) + CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      GetParameterValues(p, out scale);
+      ScaleParameter.Value = new DoubleValue(scale);
+      CovarianceFunctionParameter.Value.SetParameter(p.Skip(1).ToArray());
+    }
+
+    private void GetParameterValues(double[] p, out double scale) {
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[0]);
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale;
+      GetParameterValues(p, out scale);
+      var subCov = CovarianceFunctionParameter.Value.GetParameterizedCovarianceFunction(p.Skip(1).ToArray(), columnIndices);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => scale * subCov.Covariance(x, i, j);
+      cov.CrossCovariance = (x, xt, i, j) => scale * subCov.CrossCovariance(x, xt, i, j);
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, subCov);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, ParameterizedCovarianceFunction cov) {
+      yield return 2 * scale * cov.Covariance(x, i, j);
+      foreach (var g in cov.CovarianceGradient(x, i, j))
+        yield return scale * g;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs	(revision 9070)
@@ -0,0 +1,129 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceSquaredExponentialArd", Description = "Squared exponential covariance function with automatic relevance determination for Gaussian processes.")]
+  public sealed class CovarianceSquaredExponentialArd : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleArray> InverseLengthParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["InverseLength"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSquaredExponentialArd(bool deserializing) : base(deserializing) { }
+    private CovarianceSquaredExponentialArd(CovarianceSquaredExponentialArd original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public CovarianceSquaredExponentialArd()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the squared exponential covariance function with ARD."));
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("InverseLength", "The inverse length parameter for automatic relevance determination."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSquaredExponentialArd(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (ScaleParameter.Value != null ? 0 : 1) +
+        (InverseLengthParameter.Value != null ? 0 : numberOfVariables);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      InverseLengthParameter.Value = new DoubleArray(inverseLength);
+    }
+
+    private void GetParameterValues(double[] p, out double scale, out double[] inverseLength) {
+      int c = 0;
+      // gather parameter values
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.ToArray();
+      } else {
+        inverseLength = p.Skip(1).Select(e => 1.0 / Math.Exp(e)).ToArray();
+        c += inverseLength.Length;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSquaredExponentialArd", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double scale;
+      double[] inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                 ? 0.0
+                 : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, inverseLength);
+      return cov;
+    }
+
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, IEnumerable<int> columnIndices, double scale, double[] inverseLength) {
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+      int k = 0;
+      foreach (var columnIndex in columnIndices) {
+        double sqrDist = Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]);
+        yield return scale * Math.Exp(-d / 2.0) * sqrDist;
+        k++;
+      }
+
+      yield return 2.0 * scale * Math.Exp(-d / 2.0);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialIso.cs	(revision 9070)
@@ -0,0 +1,127 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceSquaredExponentialIso",
+    Description = "Isotropic squared exponential covariance function for Gaussian processes.")]
+  public sealed class CovarianceSquaredExponentialIso : ParameterizedNamedItem, ICovarianceFunction {
+    public IValueParameter<DoubleValue> ScaleParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Scale"]; }
+    }
+
+    public IValueParameter<DoubleValue> InverseLengthParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["InverseLength"]; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSquaredExponentialIso(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceSquaredExponentialIso(CovarianceSquaredExponentialIso original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public CovarianceSquaredExponentialIso()
+      : base() {
+      Name = ItemName;
+      Description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Scale", "The scale parameter of the isometric squared exponential covariance function."));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("InverseLength", "The inverse length parameter of the isometric squared exponential covariance function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSquaredExponentialIso(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return
+        (ScaleParameter.Value != null ? 0 : 1) +
+        (InverseLengthParameter.Value != null ? 0 : 1);
+    }
+
+    public void SetParameter(double[] p) {
+      double scale, inverseLength;
+      GetParameterValues(p, out scale, out inverseLength);
+      ScaleParameter.Value = new DoubleValue(scale);
+      InverseLengthParameter.Value = new DoubleValue(inverseLength);
+    }
+
+
+    private void GetParameterValues(double[] p, out double scale, out double inverseLength) {
+      // gather parameter values
+      int c = 0;
+      if (InverseLengthParameter.Value != null) {
+        inverseLength = InverseLengthParameter.Value.Value;
+      } else {
+        inverseLength = 1.0 / Math.Exp(p[c]);
+        c++;
+      }
+
+      if (ScaleParameter.Value != null) {
+        scale = ScaleParameter.Value.Value;
+      } else {
+        scale = Math.Exp(2 * p[c]);
+        c++;
+      }
+      if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSquaredExponentialIso", "p");
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      double inverseLength, scale;
+      GetParameterValues(p, out scale, out inverseLength);
+      // create functions
+      var cov = new ParameterizedCovarianceFunction();
+      cov.Covariance = (x, i, j) => {
+        double d = i == j
+                ? 0.0
+                : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CrossCovariance = (x, xt, i, j) => {
+        double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+        return scale * Math.Exp(-d / 2.0);
+      };
+      cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, scale, inverseLength, columnIndices);
+      return cov;
+    }
+
+    private static IEnumerable<double> GetGradient(double[,] x, int i, int j, double sf2, double inverseLength, IEnumerable<int> columnIndices) {
+      double d = i == j
+                   ? 0.0
+                   : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+      double g = Math.Exp(-d / 2.0);
+      yield return sf2 * g * d;
+      yield return 2.0 * sf2 * g;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSum.cs	(revision 9070)
@@ -0,0 +1,94 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "CovarianceSum",
+    Description = "Sum covariance function for Gaussian processes.")]
+  public sealed class CovarianceSum : Item, ICovarianceFunction {
+    [Storable]
+    private ItemList<ICovarianceFunction> terms;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<ICovarianceFunction> Terms {
+      get { return terms; }
+    }
+
+    [StorableConstructor]
+    private CovarianceSum(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private CovarianceSum(CovarianceSum original, Cloner cloner)
+      : base(original, cloner) {
+      this.terms = cloner.Clone(original.terms);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public CovarianceSum()
+      : base() {
+      this.terms = new ItemList<ICovarianceFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new CovarianceSum(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices) {
+      if (terms.Count == 0) throw new ArgumentException("at least one term is necessary for the product covariance function.");
+      var functions = new List<ParameterizedCovarianceFunction>();
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        functions.Add(t.GetParameterizedCovarianceFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+      }
+
+      var sum = new ParameterizedCovarianceFunction();
+      sum.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Sum();
+      sum.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Sum();
+      sum.CovarianceGradient = (x, i, j) => functions.Select(e => e.CovarianceGradient(x, i, j)).Aggregate(Enumerable.Concat);
+      return sum;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassification.cs	(revision 9070)
@@ -0,0 +1,196 @@
+
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Algorithms.GradientDescent;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Gaussian process least-squares classification data analysis algorithm.
+  /// </summary>
+  [Item("Gaussian Process Least-Squares Classification", "Gaussian process least-squares classification data analysis algorithm.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class GaussianProcessClassification : EngineAlgorithm, IStorableContent {
+    public string Filename { get; set; }
+
+    public override Type ProblemType { get { return typeof(IClassificationProblem); } }
+    public new IClassificationProblem Problem {
+      get { return (IClassificationProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string MinimizationIterationsParameterName = "Iterations";
+    private const string ApproximateGradientsParameterName = "ApproximateGradients";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+
+    #region parameter properties
+    public IValueParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (IValueParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    public IValueParameter<IntValue> MinimizationIterationsParameter {
+      get { return (IValueParameter<IntValue>)Parameters[MinimizationIterationsParameterName]; }
+    }
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    #endregion
+    #region properties
+    public IMeanFunction MeanFunction {
+      set { MeanFunctionParameter.Value = value; }
+      get { return MeanFunctionParameter.Value; }
+    }
+    public ICovarianceFunction CovarianceFunction {
+      set { CovarianceFunctionParameter.Value = value; }
+      get { return CovarianceFunctionParameter.Value; }
+    }
+    public int MinimizationIterations {
+      set { MinimizationIterationsParameter.Value.Value = value; }
+      get { return MinimizationIterationsParameter.Value.Value; }
+    }
+    public int Seed { get { return SeedParameter.Value.Value; } set { SeedParameter.Value.Value = value; } }
+    public bool SetSeedRandomly { get { return SetSeedRandomlyParameter.Value.Value; } set { SetSeedRandomlyParameter.Value.Value = value; } }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessClassification(bool deserializing) : base(deserializing) { }
+    private GaussianProcessClassification(GaussianProcessClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public GaussianProcessClassification()
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      Problem = new ClassificationProblem();
+
+      Parameters.Add(new ValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.", new MeanConst()));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.", new CovarianceSquaredExponentialIso()));
+      Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+
+      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
+      Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
+
+      var randomCreator = new HeuristicLab.Random.RandomCreator();
+      var gpInitializer = new GaussianProcessHyperparameterInitializer();
+      var bfgsInitializer = new LbfgsInitializer();
+      var makeStep = new LbfgsMakeStep();
+      var branch = new ConditionalBranch();
+      var modelCreator = new GaussianProcessClassificationModelCreator();
+      var updateResults = new LbfgsUpdateResults();
+      var analyzer = new LbfgsAnalyzer();
+      var finalModelCreator = new GaussianProcessClassificationModelCreator();
+      var finalAnalyzer = new LbfgsAnalyzer();
+      var solutionCreator = new GaussianProcessClassificationSolutionCreator();
+
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = gpInitializer;
+
+      gpInitializer.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      gpInitializer.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      gpInitializer.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      gpInitializer.HyperparameterParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      gpInitializer.RandomParameter.ActualName = randomCreator.RandomParameter.Name;
+      gpInitializer.Successor = bfgsInitializer;
+
+      bfgsInitializer.IterationsParameter.ActualName = MinimizationIterationsParameterName;
+      bfgsInitializer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      bfgsInitializer.Successor = makeStep;
+
+      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      makeStep.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      makeStep.Successor = branch;
+
+      branch.ConditionParameter.ActualName = makeStep.TerminationCriterionParameter.Name;
+      branch.FalseBranch = modelCreator;
+      branch.TrueBranch = finalModelCreator;
+
+      modelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      modelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      modelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      modelCreator.Successor = updateResults;
+
+      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      updateResults.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      updateResults.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      updateResults.Successor = analyzer;
+
+      analyzer.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      analyzer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      analyzer.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      analyzer.PointsTableParameter.ActualName = "Hyperparameter table";
+      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
+      analyzer.QualitiesTableParameter.ActualName = "Negative log likelihood table";
+      analyzer.Successor = makeStep;
+
+      finalModelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      finalModelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      finalModelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      finalModelCreator.HyperparameterParameter.ActualName = bfgsInitializer.PointParameter.ActualName;
+      finalModelCreator.Successor = finalAnalyzer;
+
+      finalAnalyzer.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      finalAnalyzer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      finalAnalyzer.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      finalAnalyzer.PointsTableParameter.ActualName = analyzer.PointsTableParameter.ActualName;
+      finalAnalyzer.QualityGradientsTableParameter.ActualName = analyzer.QualityGradientsTableParameter.ActualName;
+      finalAnalyzer.QualitiesTableParameter.ActualName = analyzer.QualitiesTableParameter.ActualName;
+      finalAnalyzer.Successor = solutionCreator;
+
+      solutionCreator.ModelParameter.ActualName = finalModelCreator.ModelParameter.Name;
+      solutionCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassification(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationModelCreator.cs	(revision 9070)
@@ -0,0 +1,81 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "GaussianProcessClassificationModelCreator",
+    Description = "Creates a Gaussian process model for least-squares classification given the data, the hyperparameters, a mean function, and a covariance function.")]
+  public sealed class GaussianProcessClassificationModelCreator : GaussianProcessModelCreator {
+    private const string ProblemDataParameterName = "ProblemData";
+
+    #region Parameter Properties
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IClassificationProblemData ProblemData {
+      get { return ProblemDataParameter.ActualValue; }
+    }
+    #endregion
+    [StorableConstructor]
+    private GaussianProcessClassificationModelCreator(bool deserializing) : base(deserializing) { }
+    private GaussianProcessClassificationModelCreator(GaussianProcessClassificationModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessClassificationModelCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>(ProblemDataParameterName, "The classification problem data for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassificationModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
+        return base.Apply();
+      }
+      catch (ArgumentException) { }
+      catch (alglib.alglibexception) { }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
+      return base.Apply();
+    }
+
+    public static IGaussianProcessModel Create(IClassificationProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessClassificationSolutionCreator.cs	(revision 9070)
@@ -0,0 +1,104 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "GaussianProcessClassificationSolutionCreator",
+    Description = "Creates a Gaussian process solution from a trained model.")]
+  public sealed class GaussianProcessClassificationSolutionCreator : SingleSuccessorOperator {
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ModelParameterName = "GaussianProcessClassificationModel";
+    private const string SolutionParameterName = "Solution";
+    private const string ResultsParameterName = "Results";
+    private const string TrainingAccuracyResultName = "Accuracy (training)";
+    private const string TestAccuracyResultName = "Accuracy (test)";
+
+    #region Parameter Properties
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IDiscriminantFunctionClassificationSolution> SolutionParameter {
+      get { return (ILookupParameter<IDiscriminantFunctionClassificationSolution>)Parameters[SolutionParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessClassificationSolutionCreator(bool deserializing) : base(deserializing) { }
+    private GaussianProcessClassificationSolutionCreator(GaussianProcessClassificationSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessClassificationSolutionCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IClassificationProblemData>(ProblemDataParameterName, "The classification problem data for the Gaussian process solution."));
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process classification model to use for the solution."));
+      // in & out
+      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
+      // out
+      Parameters.Add(new LookupParameter<IDiscriminantFunctionClassificationSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessClassificationSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      if (ModelParameter.ActualValue != null) {
+        var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+        m.FixParameters();
+        var data = (IClassificationProblemData)ProblemDataParameter.ActualValue.Clone();
+        var model = new DiscriminantFunctionClassificationModel(m, new NormalDistributionCutPointsThresholdCalculator());
+        model.RecalculateModelParameters(data, data.TrainingIndices);
+        var s = model.CreateDiscriminantFunctionClassificationSolution(data);
+
+        SolutionParameter.ActualValue = s;
+        var results = ResultsParameter.ActualValue;
+        if (!results.ContainsKey(SolutionParameterName)) {
+          results.Add(new Result(SolutionParameterName, "The Gaussian process classification solution", s));
+          results.Add(new Result(TrainingAccuracyResultName,
+                                 "The accuracy of the Gaussian process solution on the training partition.",
+                                 new DoubleValue(s.TrainingAccuracy)));
+          results.Add(new Result(TestAccuracyResultName,
+                                 "The accuracy of the Gaussian process solution on the test partition.",
+                                 new DoubleValue(s.TestAccuracy)));
+        } else {
+          results[SolutionParameterName].Value = s;
+          results[TrainingAccuracyResultName].Value = new DoubleValue(s.TrainingAccuracy);
+          results[TestAccuracyResultName].Value = new DoubleValue(s.TestAccuracy);
+        }
+      }
+      return base.Apply();
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs	(revision 9070)
@@ -0,0 +1,99 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "GaussianProcessHyperparameterInitializer",
+    Description = "Initializers the hyperparameter vector based on the mean function, covariance function, and number of allowed input variables.")]
+  public sealed class GaussianProcessHyperparameterInitializer : SingleSuccessorOperator {
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string HyperparameterParameterName = "Hyperparameter";
+    private const string RandomParameterName = "Random";
+
+    #region Parameter Properties
+    // in
+    public ILookupParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (ILookupParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public ILookupParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (ILookupParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    public ILookupParameter<IDataAnalysisProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IDataAnalysisProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IRandom> RandomParameter {
+      get { return (ILookupParameter<IRandom>)Parameters[RandomParameterName]; }
+    }
+    // out
+    public ILookupParameter<RealVector> HyperparameterParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
+    private ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
+    private IDataAnalysisProblemData ProblemData { get { return ProblemDataParameter.ActualValue; } }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessHyperparameterInitializer(bool deserializing) : base(deserializing) { }
+    private GaussianProcessHyperparameterInitializer(GaussianProcessHyperparameterInitializer original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessHyperparameterInitializer()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<IDataAnalysisProblemData>(ProblemDataParameterName, "The input data for the Gaussian process."));
+      Parameters.Add(new LookupParameter<IRandom>(RandomParameterName, "The pseudo random number generator to use for initializing the hyperparameter vector."));
+      // out
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The initial hyperparameter vector for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessHyperparameterInitializer(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var inputVariablesCount = ProblemData.AllowedInputVariables.Count();
+      int l = 1 + MeanFunction.GetNumberOfParameters(inputVariablesCount) +
+              CovarianceFunction.GetNumberOfParameters(inputVariablesCount);
+      var r = new RealVector(l);
+      var rand = RandomParameter.ActualValue;
+      for (int i = 0; i < r.Length; i++)
+        r[i] = rand.NextDouble() * 10 - 5;
+
+      HyperparameterParameter.ActualValue = r;
+      return base.Apply();
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs	(revision 9070)
@@ -0,0 +1,309 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a Gaussian process model.
+  /// </summary>
+  [StorableClass]
+  [Item("GaussianProcessModel", "Represents a Gaussian process posterior.")]
+  public sealed class GaussianProcessModel : NamedItem, IGaussianProcessModel {
+    [Storable]
+    private double negativeLogLikelihood;
+    public double NegativeLogLikelihood {
+      get { return negativeLogLikelihood; }
+    }
+
+    [Storable]
+    private double[] hyperparameterGradients;
+    public double[] HyperparameterGradients {
+      get {
+        var copy = new double[hyperparameterGradients.Length];
+        Array.Copy(hyperparameterGradients, copy, copy.Length);
+        return copy;
+      }
+    }
+
+    [Storable]
+    private ICovarianceFunction covarianceFunction;
+    public ICovarianceFunction CovarianceFunction {
+      get { return covarianceFunction; }
+    }
+    [Storable]
+    private IMeanFunction meanFunction;
+    public IMeanFunction MeanFunction {
+      get { return meanFunction; }
+    }
+    [Storable]
+    private string targetVariable;
+    public string TargetVariable {
+      get { return targetVariable; }
+    }
+    [Storable]
+    private string[] allowedInputVariables;
+    public string[] AllowedInputVariables {
+      get { return allowedInputVariables; }
+    }
+
+    [Storable]
+    private double[] alpha;
+    [Storable]
+    private double sqrSigmaNoise;
+    public double SigmaNoise {
+      get { return Math.Sqrt(sqrSigmaNoise); }
+    }
+
+    [Storable]
+    private double[] meanParameter;
+    [Storable]
+    private double[] covarianceParameter;
+
+    [Storable]
+    private double[,] l;
+
+    [Storable]
+    private double[,] x;
+    [Storable]
+    private Scaling inputScaling;
+
+
+    [StorableConstructor]
+    private GaussianProcessModel(bool deserializing) : base(deserializing) { }
+    private GaussianProcessModel(GaussianProcessModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.meanFunction = cloner.Clone(original.meanFunction);
+      this.covarianceFunction = cloner.Clone(original.covarianceFunction);
+      this.inputScaling = cloner.Clone(original.inputScaling);
+      this.negativeLogLikelihood = original.negativeLogLikelihood;
+      this.targetVariable = original.targetVariable;
+      this.sqrSigmaNoise = original.sqrSigmaNoise;
+      if (original.meanParameter != null) {
+        this.meanParameter = (double[])original.meanParameter.Clone();
+      }
+      if (original.covarianceParameter != null) {
+        this.covarianceParameter = (double[])original.covarianceParameter.Clone();
+      }
+
+      // shallow copies of arrays because they cannot be modified
+      this.allowedInputVariables = original.allowedInputVariables;
+      this.alpha = original.alpha;
+      this.l = original.l;
+      this.x = original.x;
+    }
+    public GaussianProcessModel(Dataset ds, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows,
+      IEnumerable<double> hyp, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.meanFunction = (IMeanFunction)meanFunction.Clone();
+      this.covarianceFunction = (ICovarianceFunction)covarianceFunction.Clone();
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+
+
+      int nVariables = this.allowedInputVariables.Length;
+      meanParameter = hyp
+        .Take(this.meanFunction.GetNumberOfParameters(nVariables))
+        .ToArray();
+
+      covarianceParameter = hyp.Skip(this.meanFunction.GetNumberOfParameters(nVariables))
+                                             .Take(this.covarianceFunction.GetNumberOfParameters(nVariables))
+                                             .ToArray();
+      sqrSigmaNoise = Math.Exp(2.0 * hyp.Last());
+
+      CalculateModel(ds, rows);
+    }
+
+    private void CalculateModel(Dataset ds, IEnumerable<int> rows) {
+      inputScaling = new Scaling(ds, allowedInputVariables, rows);
+      x = AlglibUtil.PrepareAndScaleInputMatrix(ds, allowedInputVariables, rows, inputScaling);
+      var y = ds.GetDoubleValues(targetVariable, rows);
+
+      int n = x.GetLength(0);
+      l = new double[n, n];
+
+      // calculate means and covariances
+      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, x.GetLength(1)));
+      double[] m = Enumerable.Range(0, x.GetLength(0))
+        .Select(r => mean.Mean(x, r))
+        .ToArray();
+
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, x.GetLength(1)));
+      for (int i = 0; i < n; i++) {
+        for (int j = i; j < n; j++) {
+          l[j, i] = cov.Covariance(x, i, j) / sqrSigmaNoise;
+          if (j == i) l[j, i] += 1.0;
+        }
+      }
+
+
+      // cholesky decomposition
+      int info;
+      alglib.densesolverreport denseSolveRep;
+
+      var res = alglib.trfac.spdmatrixcholesky(ref l, n, false);
+      if (!res) throw new ArgumentException("Matrix is not positive semidefinite");
+
+      // calculate sum of diagonal elements for likelihood
+      double diagSum = Enumerable.Range(0, n).Select(i => Math.Log(l[i, i])).Sum();
+
+      // solve for alpha
+      double[] ym = y.Zip(m, (a, b) => a - b).ToArray();
+
+      alglib.spdmatrixcholeskysolve(l, n, false, ym, out info, out denseSolveRep, out alpha);
+      for (int i = 0; i < alpha.Length; i++)
+        alpha[i] = alpha[i] / sqrSigmaNoise;
+      negativeLogLikelihood = 0.5 * Util.ScalarProd(ym, alpha) + diagSum + (n / 2.0) * Math.Log(2.0 * Math.PI * sqrSigmaNoise);
+
+      // derivatives
+      int nAllowedVariables = x.GetLength(1);
+
+      alglib.matinvreport matInvRep;
+      double[,] lCopy = new double[l.GetLength(0), l.GetLength(1)];
+      Array.Copy(l, lCopy, lCopy.Length);
+
+      alglib.spdmatrixcholeskyinverse(ref lCopy, n, false, out info, out matInvRep);
+      if (info != 1) throw new ArgumentException("Can't invert matrix to calculate gradients.");
+      for (int i = 0; i < n; i++) {
+        for (int j = 0; j <= i; j++)
+          lCopy[i, j] = lCopy[i, j] / sqrSigmaNoise - alpha[i] * alpha[j];
+      }
+
+      double noiseGradient = sqrSigmaNoise * Enumerable.Range(0, n).Select(i => lCopy[i, i]).Sum();
+
+      double[] meanGradients = new double[meanFunction.GetNumberOfParameters(nAllowedVariables)];
+      for (int k = 0; k < meanGradients.Length; k++) {
+        var meanGrad = Enumerable.Range(0, alpha.Length)
+        .Select(r => mean.Gradient(x, r, k));
+        meanGradients[k] = -Util.ScalarProd(meanGrad, alpha);
+      }
+
+      double[] covGradients = new double[covarianceFunction.GetNumberOfParameters(nAllowedVariables)];
+      if (covGradients.Length > 0) {
+        for (int i = 0; i < n; i++) {
+          for (int j = 0; j < i; j++) {
+            var g = cov.CovarianceGradient(x, i, j).ToArray();
+            for (int k = 0; k < covGradients.Length; k++) {
+              covGradients[k] += lCopy[i, j] * g[k];
+            }
+          }
+
+          var gDiag = cov.CovarianceGradient(x, i, i).ToArray();
+          for (int k = 0; k < covGradients.Length; k++) {
+            // diag
+            covGradients[k] += 0.5 * lCopy[i, i] * gDiag[k];
+          }
+        }
+      }
+
+      hyperparameterGradients =
+        meanGradients
+        .Concat(covGradients)
+        .Concat(new double[] { noiseGradient }).ToArray();
+
+    }
+
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessModel(this, cloner);
+    }
+
+    // is called by the solution creator to set all parameter values of the covariance and mean function
+    // to the optimized values (necessary to make the values visible in the GUI)
+    public void FixParameters() {
+      covarianceFunction.SetParameter(covarianceParameter);
+      meanFunction.SetParameter(meanParameter);
+      covarianceParameter = new double[0];
+      meanParameter = new double[0];
+    }
+
+    #region IRegressionModel Members
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      return GetEstimatedValuesHelper(dataset, rows);
+    }
+    public GaussianProcessRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new GaussianProcessRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    #endregion
+
+
+    private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
+      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
+      int newN = newX.GetLength(0);
+      int n = x.GetLength(0);
+      var Ks = new double[newN, n];
+      var mean = meanFunction.GetParameterizedMeanFunction(meanParameter, Enumerable.Range(0, newX.GetLength(1)));
+      var ms = Enumerable.Range(0, newX.GetLength(0))
+      .Select(r => mean.Mean(newX, r))
+      .ToArray();
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, newX.GetLength(1)));
+      for (int i = 0; i < newN; i++) {
+        for (int j = 0; j < n; j++) {
+          Ks[i, j] = cov.CrossCovariance(x, newX, j, i);
+        }
+      }
+
+      return Enumerable.Range(0, newN)
+        .Select(i => ms[i] + Util.ScalarProd(Util.GetRow(Ks, i), alpha));
+    }
+
+    public IEnumerable<double> GetEstimatedVariance(Dataset dataset, IEnumerable<int> rows) {
+      var newX = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, inputScaling);
+      int newN = newX.GetLength(0);
+      int n = x.GetLength(0);
+
+      var kss = new double[newN];
+      double[,] sWKs = new double[n, newN];
+      var cov = covarianceFunction.GetParameterizedCovarianceFunction(covarianceParameter, Enumerable.Range(0, newX.GetLength(1)));
+
+      // for stddev 
+      for (int i = 0; i < newN; i++)
+        kss[i] = cov.Covariance(newX, i, i);
+
+      for (int i = 0; i < newN; i++) {
+        for (int j = 0; j < n; j++) {
+          sWKs[j, i] = cov.CrossCovariance(x, newX, j, i) / Math.Sqrt(sqrSigmaNoise);
+        }
+      }
+
+      // for stddev 
+      alglib.ablas.rmatrixlefttrsm(n, newN, l, 0, 0, false, false, 0, ref sWKs, 0, 0);
+
+      for (int i = 0; i < newN; i++) {
+        var sumV = Util.ScalarProd(Util.GetCol(sWKs, i), Util.GetCol(sWKs, i));
+        kss[i] -= sumV;
+        if (kss[i] < 0) kss[i] = 0;
+      }
+      return kss;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs	(revision 9070)
@@ -0,0 +1,86 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  // base class for GaussianProcessModelCreators (specific for classification and regression)
+  public abstract class GaussianProcessModelCreator : SingleSuccessorOperator {
+    private const string HyperparameterParameterName = "Hyperparameter";
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string ModelParameterName = "Model";
+    private const string NegativeLogLikelihoodParameterName = "NegativeLogLikelihood";
+    private const string HyperparameterGradientsParameterName = "HyperparameterGradients";
+
+    #region Parameter Properties
+    // in
+    public ILookupParameter<RealVector> HyperparameterParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
+    }
+    public ILookupParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (ILookupParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public ILookupParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (ILookupParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    // out
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<RealVector> HyperparameterGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterGradientsParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NegativeLogLikelihoodParameterName]; }
+    }
+
+    #endregion
+
+    #region Properties
+    protected RealVector Hyperparameter { get { return HyperparameterParameter.ActualValue; } }
+    protected IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
+    protected ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
+    #endregion
+
+    [StorableConstructor]
+    protected GaussianProcessModelCreator(bool deserializing) : base(deserializing) { }
+    protected GaussianProcessModelCreator(GaussianProcessModelCreator original, Cloner cloner) : base(original, cloner) { }
+    protected GaussianProcessModelCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The hyperparameters for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function for the Gaussian process model."));
+      Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
+      // out
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The resulting Gaussian process model"));
+      Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
+      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs	(revision 9070)
@@ -0,0 +1,196 @@
+
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Algorithms.GradientDescent;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  ///Gaussian process regression data analysis algorithm.
+  /// </summary>
+  [Item("Gaussian Process Regression", "Gaussian process regression data analysis algorithm.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class GaussianProcessRegression : EngineAlgorithm, IStorableContent {
+    public string Filename { get; set; }
+
+    public override Type ProblemType { get { return typeof(IRegressionProblem); } }
+    public new IRegressionProblem Problem {
+      get { return (IRegressionProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
+    private const string MeanFunctionParameterName = "MeanFunction";
+    private const string CovarianceFunctionParameterName = "CovarianceFunction";
+    private const string MinimizationIterationsParameterName = "Iterations";
+    private const string ApproximateGradientsParameterName = "ApproximateGradients";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+
+    #region parameter properties
+    public IValueParameter<IMeanFunction> MeanFunctionParameter {
+      get { return (IValueParameter<IMeanFunction>)Parameters[MeanFunctionParameterName]; }
+    }
+    public IValueParameter<ICovarianceFunction> CovarianceFunctionParameter {
+      get { return (IValueParameter<ICovarianceFunction>)Parameters[CovarianceFunctionParameterName]; }
+    }
+    public IValueParameter<IntValue> MinimizationIterationsParameter {
+      get { return (IValueParameter<IntValue>)Parameters[MinimizationIterationsParameterName]; }
+    }
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    #endregion
+    #region properties
+    public IMeanFunction MeanFunction {
+      set { MeanFunctionParameter.Value = value; }
+      get { return MeanFunctionParameter.Value; }
+    }
+    public ICovarianceFunction CovarianceFunction {
+      set { CovarianceFunctionParameter.Value = value; }
+      get { return CovarianceFunctionParameter.Value; }
+    }
+    public int MinimizationIterations {
+      set { MinimizationIterationsParameter.Value.Value = value; }
+      get { return MinimizationIterationsParameter.Value.Value; }
+    }
+    public int Seed { get { return SeedParameter.Value.Value; } set { SeedParameter.Value.Value = value; } }
+    public bool SetSeedRandomly { get { return SetSeedRandomlyParameter.Value.Value; } set { SetSeedRandomlyParameter.Value.Value = value; } }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessRegression(bool deserializing) : base(deserializing) { }
+    private GaussianProcessRegression(GaussianProcessRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public GaussianProcessRegression()
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      Problem = new RegressionProblem();
+
+      Parameters.Add(new ValueParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function to use.", new MeanConst()));
+      Parameters.Add(new ValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.", new CovarianceSquaredExponentialIso()));
+      Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+
+      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
+      Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
+
+      var randomCreator = new HeuristicLab.Random.RandomCreator();
+      var gpInitializer = new GaussianProcessHyperparameterInitializer();
+      var bfgsInitializer = new LbfgsInitializer();
+      var makeStep = new LbfgsMakeStep();
+      var branch = new ConditionalBranch();
+      var modelCreator = new GaussianProcessRegressionModelCreator();
+      var updateResults = new LbfgsUpdateResults();
+      var analyzer = new LbfgsAnalyzer();
+      var finalModelCreator = new GaussianProcessRegressionModelCreator();
+      var finalAnalyzer = new LbfgsAnalyzer();
+      var solutionCreator = new GaussianProcessRegressionSolutionCreator();
+
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = gpInitializer;
+
+      gpInitializer.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      gpInitializer.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      gpInitializer.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      gpInitializer.HyperparameterParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      gpInitializer.RandomParameter.ActualName = randomCreator.RandomParameter.Name;
+      gpInitializer.Successor = bfgsInitializer;
+
+      bfgsInitializer.IterationsParameter.ActualName = MinimizationIterationsParameterName;
+      bfgsInitializer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      bfgsInitializer.Successor = makeStep;
+
+      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      makeStep.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      makeStep.Successor = branch;
+
+      branch.ConditionParameter.ActualName = makeStep.TerminationCriterionParameter.Name;
+      branch.FalseBranch = modelCreator;
+      branch.TrueBranch = finalModelCreator;
+
+      modelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      modelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      modelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      modelCreator.Successor = updateResults;
+
+      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      updateResults.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      updateResults.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      updateResults.Successor = analyzer;
+
+      analyzer.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      analyzer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      analyzer.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      analyzer.PointsTableParameter.ActualName = "Hyperparameter table";
+      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
+      analyzer.QualitiesTableParameter.ActualName = "Negative log likelihood table";
+      analyzer.Successor = makeStep;
+
+      finalModelCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+      finalModelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
+      finalModelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
+      finalModelCreator.HyperparameterParameter.ActualName = bfgsInitializer.PointParameter.ActualName;
+      finalModelCreator.Successor = finalAnalyzer;
+
+      finalAnalyzer.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
+      finalAnalyzer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
+      finalAnalyzer.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
+      finalAnalyzer.PointsTableParameter.ActualName = analyzer.PointsTableParameter.ActualName;
+      finalAnalyzer.QualityGradientsTableParameter.ActualName = analyzer.QualityGradientsTableParameter.ActualName;
+      finalAnalyzer.QualitiesTableParameter.ActualName = analyzer.QualitiesTableParameter.ActualName;
+      finalAnalyzer.Successor = solutionCreator;
+
+      solutionCreator.ModelParameter.ActualName = finalModelCreator.ModelParameter.Name;
+      solutionCreator.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegression(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs	(revision 9070)
@@ -0,0 +1,81 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "GaussianProcessRegressionModelCreator",
+    Description = "Creates a Gaussian process model for regression given the data, the hyperparameters, a mean function, and a covariance function.")]
+  public sealed class GaussianProcessRegressionModelCreator : GaussianProcessModelCreator {
+    private const string ProblemDataParameterName = "ProblemData";
+
+    #region Parameter Properties
+    public ILookupParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    #endregion
+
+    #region Properties
+    private IRegressionProblemData ProblemData {
+      get { return ProblemDataParameter.ActualValue; }
+    }
+    #endregion
+    [StorableConstructor]
+    private GaussianProcessRegressionModelCreator(bool deserializing) : base(deserializing) { }
+    private GaussianProcessRegressionModelCreator(GaussianProcessRegressionModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessRegressionModelCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IRegressionProblemData>(ProblemDataParameterName, "The regression problem data for the Gaussian process model."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      try {
+        var model = Create(ProblemData, Hyperparameter.ToArray(), MeanFunction, CovarianceFunction);
+        ModelParameter.ActualValue = model;
+        NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
+        HyperparameterGradientsParameter.ActualValue = new RealVector(model.HyperparameterGradients);
+        return base.Apply();
+      }
+      catch (ArgumentException) { }
+      catch (alglib.alglibexception) { }
+      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(1E300);
+      HyperparameterGradientsParameter.ActualValue = new RealVector(Hyperparameter.Count());
+      return base.Apply();
+    }
+
+    public static IGaussianProcessModel Create(IRegressionProblemData problemData, double[] hyperparameter, IMeanFunction meanFunction, ICovarianceFunction covarianceFunction) {
+      return new GaussianProcessModel(problemData.Dataset, problemData.TargetVariable, problemData.AllowedInputVariables, problemData.TrainingIndices, hyperparameter, meanFunction, covarianceFunction);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,94 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a Gaussian process solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("GaussianProcessRegressionSolution", "Represents a Gaussian process solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class GaussianProcessRegressionSolution : RegressionSolution, IGaussianProcessSolution {
+    private new readonly Dictionary<int, double> evaluationCache;
+
+    public new IGaussianProcessModel Model {
+      get { return (IGaussianProcessModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private GaussianProcessRegressionSolution(bool deserializing)
+      : base(deserializing) {
+      evaluationCache = new Dictionary<int, double>();
+
+    }
+    private GaussianProcessRegressionSolution(GaussianProcessRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+      evaluationCache = new Dictionary<int, double>(original.evaluationCache);
+    }
+    public GaussianProcessRegressionSolution(IGaussianProcessModel model, IRegressionProblemData problemData)
+      : base(model, problemData) {
+
+      evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionSolution(this, cloner);
+    }
+
+    public IEnumerable<double> EstimatedVariance {
+      get { return GetEstimatedVariance(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
+    }
+    public IEnumerable<double> EstimatedTrainingVariance {
+      get { return GetEstimatedVariance(ProblemData.TrainingIndices); }
+    }
+    public IEnumerable<double> EstimatedTestVariance {
+      get { return GetEstimatedVariance(ProblemData.TestIndices); }
+    }
+
+    public IEnumerable<double> GetEstimatedVariance(IEnumerable<int> rows) {
+      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
+      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+      var valuesEnumerator = Model.GetEstimatedVariance(ProblemData.Dataset, rowsToEvaluate).GetEnumerator();
+
+      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
+      }
+
+      return rows.Select(row => evaluationCache[row]);
+    }
+
+    protected override void OnModelChanged() {
+      evaluationCache.Clear();
+      base.OnModelChanged();
+    }
+    protected override void OnProblemDataChanged() {
+      evaluationCache.Clear();
+      base.OnProblemDataChanged();
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionSolutionCreator.cs	(revision 9070)
@@ -0,0 +1,103 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "GaussianProcessRegressionSolutionCreator",
+    Description = "Creates a Gaussian process solution from a trained model.")]
+  public sealed class GaussianProcessRegressionSolutionCreator : SingleSuccessorOperator {
+    private const string ProblemDataParameterName = "ProblemData";
+    private const string ModelParameterName = "GaussianProcessRegressionModel";
+    private const string SolutionParameterName = "Solution";
+    private const string ResultsParameterName = "Results";
+    private const string TrainingRSquaredResultName = "Training R²";
+    private const string TestRSquaredResultName = "Test R²";
+
+    #region Parameter Properties
+    public ILookupParameter<IRegressionProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessSolution> SolutionParameter {
+      get { return (ILookupParameter<IGaussianProcessSolution>)Parameters[SolutionParameterName]; }
+    }
+    public ILookupParameter<IGaussianProcessModel> ModelParameter {
+      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
+    }
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private GaussianProcessRegressionSolutionCreator(bool deserializing) : base(deserializing) { }
+    private GaussianProcessRegressionSolutionCreator(GaussianProcessRegressionSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public GaussianProcessRegressionSolutionCreator()
+      : base() {
+      // in
+      Parameters.Add(new LookupParameter<IRegressionProblemData>(ProblemDataParameterName, "The regression problem data for the Gaussian process solution."));
+      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The Gaussian process regression model to use for the solution."));
+      // in & out
+      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
+      // out
+      Parameters.Add(new LookupParameter<IGaussianProcessSolution>(SolutionParameterName, "The produced Gaussian process solution."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GaussianProcessRegressionSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      if (ModelParameter.ActualValue != null) {
+        var m = (IGaussianProcessModel)ModelParameter.ActualValue.Clone();
+        m.FixParameters();
+        var data = (IRegressionProblemData)ProblemDataParameter.ActualValue.Clone();
+        var s = new GaussianProcessRegressionSolution(m, data);
+
+
+        SolutionParameter.ActualValue = s;
+        var results = ResultsParameter.ActualValue;
+        if (!results.ContainsKey(SolutionParameterName)) {
+          results.Add(new Result(SolutionParameterName, "The Gaussian process regression solution", s));
+          results.Add(new Result(TrainingRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the training partition.",
+                                 new DoubleValue(s.TrainingRSquared)));
+          results.Add(new Result(TestRSquaredResultName,
+                                 "The Pearson's R² of the Gaussian process solution on the test partition.",
+                                 new DoubleValue(s.TestRSquared)));
+        } else {
+          results[SolutionParameterName].Value = s;
+          results[TrainingRSquaredResultName].Value = new DoubleValue(s.TrainingRSquared);
+          results[TestRSquaredResultName].Value = new DoubleValue(s.TestRSquared);
+        }
+      }
+      return base.Apply();
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/ICovarianceFunction.cs	(revision 9070)
@@ -0,0 +1,44 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+
+  public delegate double CovarianceFunctionDelegate(double[,] x, int i, int j);
+  public delegate double CrossCovarianceFunctionDelegate(double[,] x, double[,] xt, int i, int j);
+  public delegate IEnumerable<double> CovarianceGradientFunctionDelegate(double[,] x, int i, int j);
+
+  public class ParameterizedCovarianceFunction {
+    public CovarianceFunctionDelegate Covariance { get; set; }
+    public CrossCovarianceFunctionDelegate CrossCovariance { get; set; }
+    public CovarianceGradientFunctionDelegate CovarianceGradient { get; set; }
+  }
+
+  public interface ICovarianceFunction : IItem {
+    int GetNumberOfParameters(int numberOfVariables);
+    void SetParameter(double[] p);
+    ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, IEnumerable<int> columnIndices);
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/IMeanFunction.cs	(revision 9070)
@@ -0,0 +1,40 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public delegate double MeanFunctionDelegate(double[,] x, int row);
+  public delegate double MeanGradientDelegate(double[,] x, int row, int k);
+
+  public class ParameterizedMeanFunction {
+    public MeanFunctionDelegate Mean { get; set; }
+    public MeanGradientDelegate Gradient { get; set; }
+  }
+
+  public interface IMeanFunction : IItem {
+    int GetNumberOfParameters(int numberOfVariables);
+    void SetParameter(double[] p);
+    ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices);
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanConst.cs	(revision 9070)
@@ -0,0 +1,90 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanConst", Description = "Constant mean function for Gaussian processes.")]
+  public sealed class MeanConst : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<DoubleValue> ValueParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters["Value"]; }
+    }
+
+    [StorableConstructor]
+    private MeanConst(bool deserializing) : base(deserializing) { }
+    private MeanConst(MeanConst original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanConst()
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("Value", "The constant value for the constant mean function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanConst(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return ValueParameter.Value != null ? 0 : 1;
+    }
+
+    public void SetParameter(double[] p) {
+      double c;
+      GetParameters(p, out c);
+      ValueParameter.Value = new DoubleValue(c);
+    }
+
+    private void GetParameters(double[] p, out double c) {
+      if (ValueParameter.Value == null) {
+        c = p[0];
+      } else {
+        if (p.Length > 0)
+          throw new ArgumentException(
+            "The length of the parameter vector does not match the number of free parameters for the constant mean function.",
+            "p");
+        c = ValueParameter.Value.Value;
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+      double c;
+      GetParameters(p, out c);
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => c;
+      mf.Gradient = (x, i, k) => {
+        if (k > 0) throw new ArgumentException();
+        return 1.0;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanLinear.cs	(revision 9070)
@@ -0,0 +1,89 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanLinear", Description = "Linear mean function for Gaussian processes.")]
+  public sealed class MeanLinear : ParameterizedNamedItem, IMeanFunction {
+    public IValueParameter<DoubleArray> WeightsParameter {
+      get { return (IValueParameter<DoubleArray>)Parameters["Weights"]; }
+    }
+
+    [StorableConstructor]
+    private MeanLinear(bool deserializing) : base(deserializing) { }
+    private MeanLinear(MeanLinear original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanLinear()
+      : base() {
+      Parameters.Add(new OptionalValueParameter<DoubleArray>("Weights", "The weights parameter for the linear mean function."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanLinear(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return WeightsParameter.Value != null ? 0 : numberOfVariables;
+    }
+
+    public void SetParameter(double[] p) {
+      double[] weights;
+      GetParameter(p, out weights);
+      WeightsParameter.Value = new DoubleArray(weights);
+    }
+
+    public void GetParameter(double[] p, out double[] weights) {
+      if (WeightsParameter.Value == null) {
+        weights = p;
+      } else {
+        if (p.Length != 0) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for the linear mean function.", "p");
+        weights = WeightsParameter.Value.ToArray();
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+      double[] weights;
+      int[] columns = columnIndices.ToArray();
+      GetParameter(p, out weights);
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => {
+        // sanity check
+        if (weights.Length != columns.Length) throw new ArgumentException("The number of rparameters must match the number of variables for the linear mean function.");
+        return Util.ScalarProd(weights, Util.GetRow(x, i, columns));
+      };
+      mf.Gradient = (x, i, k) => {
+        if (k > columns.Length) throw new ArgumentException();
+        return x[i, columns[k]];
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanProduct.cs	(revision 9070)
@@ -0,0 +1,113 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanProduct", Description = "Product of mean functions for Gaussian processes.")]
+  public sealed class MeanProduct : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> factors;
+
+    [Storable]
+    private int numberOfVariables;
+
+    public ItemList<IMeanFunction> Factors {
+      get { return factors; }
+    }
+
+    [StorableConstructor]
+    private MeanProduct(bool deserializing)
+      : base(deserializing) {
+    }
+
+    private MeanProduct(MeanProduct original, Cloner cloner)
+      : base(original, cloner) {
+      this.factors = cloner.Clone(original.factors);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+
+    public MeanProduct() {
+      this.factors = new ItemList<IMeanFunction>();
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanProduct(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return factors.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in factors) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+      var factorMf = new List<ParameterizedMeanFunction>();
+      int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
+      int[] factorIndexMap = new int[totalNumberOfParameters]; // maps k-th hyperparameter to the correct mean-term
+      int[] hyperParameterIndexMap = new int[totalNumberOfParameters]; // maps k-th hyperparameter to the l-th hyperparameter of the correct mean-term
+      int c = 0;
+      // get the parameterized mean function for each term
+      for (int factorIndex = 0; factorIndex < factors.Count; factorIndex++) {
+        var numberOfParameters = factors[factorIndex].GetNumberOfParameters(numberOfVariables);
+        factorMf.Add(factors[factorIndex].GetParameterizedMeanFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+
+        for (int hyperParameterIndex = 0; hyperParameterIndex < numberOfParameters; hyperParameterIndex++) {
+          factorIndexMap[c] = factorIndex;
+          hyperParameterIndexMap[c] = hyperParameterIndex;
+          c++;
+        }
+      }
+
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => factorMf.Select(t => t.Mean(x, i)).Aggregate((a, b) => a * b);
+      mf.Gradient = (x, i, k) => {
+        double result = 1.0;
+        int hyperParameterFactorIndex = factorIndexMap[k];
+        for (int factorIndex = 0; factorIndex < factors.Count; factorIndex++) {
+          if (factorIndex == hyperParameterFactorIndex) {
+            // multiply gradient
+            result *= factorMf[factorIndex].Gradient(x, i, hyperParameterIndexMap[k]);
+          } else {
+            // multiply mean
+            result *= factorMf[factorIndex].Mean(x, i);
+          }
+        }
+        return result;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanSum.cs	(revision 9070)
@@ -0,0 +1,97 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanSum", Description = "Sum of mean functions for Gaussian processes.")]
+  public sealed class MeanSum : Item, IMeanFunction {
+    [Storable]
+    private ItemList<IMeanFunction> terms;
+
+    [Storable]
+    private int numberOfVariables;
+    public ItemList<IMeanFunction> Terms {
+      get { return terms; }
+    }
+
+    [StorableConstructor]
+    private MeanSum(bool deserializing) : base(deserializing) { }
+    private MeanSum(MeanSum original, Cloner cloner)
+      : base(original, cloner) {
+      this.terms = cloner.Clone(original.terms);
+      this.numberOfVariables = original.numberOfVariables;
+    }
+    public MeanSum() {
+      this.terms = new ItemList<IMeanFunction>();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanSum(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      this.numberOfVariables = numberOfVariables;
+      return terms.Select(t => t.GetNumberOfParameters(numberOfVariables)).Sum();
+    }
+
+    public void SetParameter(double[] p) {
+      int offset = 0;
+      foreach (var t in terms) {
+        var numberOfParameters = t.GetNumberOfParameters(numberOfVariables);
+        t.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+        offset += numberOfParameters;
+      }
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+      var termMf = new List<ParameterizedMeanFunction>();
+      int totalNumberOfParameters = GetNumberOfParameters(numberOfVariables);
+      int[] termIndexMap = new int[totalNumberOfParameters]; // maps k-th parameter to the correct mean-term
+      int[] hyperParameterIndexMap = new int[totalNumberOfParameters]; // maps k-th parameter to the l-th parameter of the correct mean-term
+      int c = 0;
+      // get the parameterized mean function for each term
+      for (int termIndex = 0; termIndex < terms.Count; termIndex++) {
+        var numberOfParameters = terms[termIndex].GetNumberOfParameters(numberOfVariables);
+        termMf.Add(terms[termIndex].GetParameterizedMeanFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+        p = p.Skip(numberOfParameters).ToArray();
+
+        for (int hyperParameterIndex = 0; hyperParameterIndex < numberOfParameters; hyperParameterIndex++) {
+          termIndexMap[c] = termIndex;
+          hyperParameterIndexMap[c] = hyperParameterIndex;
+          c++;
+        }
+      }
+
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => termMf.Select(t => t.Mean(x, i)).Sum();
+      mf.Gradient = (x, i, k) => {
+        return termMf[termIndexMap[k]].Gradient(x, i, hyperParameterIndexMap[k]);
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/MeanFunctions/MeanZero.cs	(revision 9070)
@@ -0,0 +1,64 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "MeanZero", Description = "Constant zero mean function for Gaussian processes.")]
+  public sealed class MeanZero : Item, IMeanFunction {
+    [StorableConstructor]
+    private MeanZero(bool deserializing) : base(deserializing) { }
+    private MeanZero(MeanZero original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MeanZero() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MeanZero(this, cloner);
+    }
+
+    public int GetNumberOfParameters(int numberOfVariables) {
+      return 0;
+    }
+
+    public void SetParameter(double[] p) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for zero mean function.", "p");
+    }
+
+    public ParameterizedMeanFunction GetParameterizedMeanFunction(double[] p, IEnumerable<int> columnIndices) {
+      if (p.Length > 0) throw new ArgumentException("No parameters allowed for zero mean function.", "p");
+      var mf = new ParameterizedMeanFunction();
+      mf.Mean = (x, i) => 0.0;
+      mf.Gradient = (x, i, k) => {
+        if (k > 0)
+          throw new ArgumentException();
+        return 0.0;
+      };
+      return mf;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/Util.cs	(revision 9070)
@@ -0,0 +1,118 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal static class Util {
+    public static double ScalarProd(IEnumerable<double> v, IEnumerable<double> u) {
+      return v.Zip(u, (vi, ui) => vi * ui).Sum();
+    }
+
+    public static double SqrDist(IEnumerable<double> x, IEnumerable<double> y) {
+      return x.Zip(y, (a, b) => (a - b) * (a - b)).Sum();
+    }
+
+    public static double SqrDist(double x, double y) {
+      double d = x - y;
+      return d * d;
+    }
+
+    public static double SqrDist(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      return SqrDist(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      double ss = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int columnIndex in columnIndices) {
+        double d = x[i, columnIndex] - xt[j, columnIndex];
+        ss += d * d;
+      }
+      return scale * scale * ss;
+    }
+
+    public static double SqrDist(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      return SqrDist(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double SqrDist(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      double ss = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      int scaleIndex = 0;
+      foreach (int columnIndex in columnIndices) {
+        double d = x[i, columnIndex] - xt[j, columnIndex];
+        ss += d * d * scale[scaleIndex] * scale[scaleIndex];
+        scaleIndex++;
+      }
+      // must be at the end of scale after iterating over columnIndices
+      if (scaleIndex != scale.Length)
+        throw new ArgumentException("Lengths of scales and covariance functions does not match.");
+      return ss;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      return ScalarProd(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double scale = 1.0, IEnumerable<int> columnIndices = null) {
+      double sum = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      foreach (int columnIndex in columnIndices) {
+        sum += x[i, columnIndex] * xt[j, columnIndex];
+      }
+      return scale * scale * sum;
+    }
+    public static double ScalarProd(double[,] x, int i, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      return ScalarProd(x, i, x, j, scale, columnIndices);
+    }
+
+    public static double ScalarProd(double[,] x, int i, double[,] xt, int j, double[] scale, IEnumerable<int> columnIndices = null) {
+      double sum = 0.0;
+      if (columnIndices == null) columnIndices = Enumerable.Range(0, x.GetLength(1));
+      int scaleIndex = 0;
+      foreach (int columnIndex in columnIndices) {
+        sum += x[i, columnIndex] * scale[scaleIndex] * xt[j, columnIndex] * scale[scaleIndex];
+        scaleIndex++;
+      }
+      // must be at the end of scale after iterating over columnIndices
+      if (scaleIndex != scale.Length)
+        throw new ArgumentException("Lengths of scales and covariance functions does not match.");
+
+      return sum;
+    }
+
+    public static IEnumerable<double> GetRow(double[,] x, int r) {
+      int cols = x.GetLength(1);
+      return GetRow(x, r, Enumerable.Range(0, cols));
+    }
+    public static IEnumerable<double> GetRow(double[,] x, int r, IEnumerable<int> columnIndices) {
+      return columnIndices.Select(c => x[r, c]);
+    }
+    public static IEnumerable<double> GetCol(double[,] x, int c) {
+      int rows = x.GetLength(0);
+      return Enumerable.Range(0, rows).Select(r => x[r, c]);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 9070)
@@ -0,0 +1,415 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>9.0.30729</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{2E782078-FA81-4B70-B56F-74CE38DAC6C8}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.DataAnalysis</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.DataAnalysis-3.4</AssemblyName>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>HeuristicLab.snk</AssemblyOriginatorKeyFile>
+    <FileUpgradeFlags>
+    </FileUpgradeFlags>
+    <OldToolsVersion>3.5</OldToolsVersion>
+    <UpgradeBackupLocation>
+    </UpgradeBackupLocation>
+    <StartupObject>
+    </StartupObject>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <TargetFrameworkProfile>
+    </TargetFrameworkProfile>
+    <PublishUrl>publish\</PublishUrl>
+    <Install>true</Install>
+    <InstallFrom>Disk</InstallFrom>
+    <UpdateEnabled>false</UpdateEnabled>
+    <UpdateMode>Foreground</UpdateMode>
+    <UpdateInterval>7</UpdateInterval>
+    <UpdateIntervalUnits>Days</UpdateIntervalUnits>
+    <UpdatePeriodically>false</UpdatePeriodically>
+    <UpdateRequired>false</UpdateRequired>
+    <MapFileExtensions>true</MapFileExtensions>
+    <ApplicationRevision>0</ApplicationRevision>
+    <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
+    <IsWebBootstrapper>false</IsWebBootstrapper>
+    <UseApplicationTrust>false</UseApplicationTrust>
+    <BootstrapperEnabled>true</BootstrapperEnabled>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <DocumentationFile>
+    </DocumentationFile>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <DocumentationFile>
+    </DocumentationFile>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
+    <OutputPath>$(SolutionDir)\bin\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <DocumentationFile>
+    </DocumentationFile>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.6.0, Version=3.6.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\bin\ALGLIB-3.6.0.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="LibSVM-3.12, Version=3.12.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <HintPath>..\..\bin\LibSVM-3.12.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Drawing" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CrossValidation.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceConst.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceLinear.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceLinearArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceMask.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceMaternIso.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceNoise.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovariancePeriodic.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceProduct.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceRationalQuadraticArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceRationalQuadraticIso.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceScale.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSquaredExponentialArd.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSquaredExponentialIso.cs" />
+    <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSum.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\GaussianProcessClassificationSolutionCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassificationModelCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessClassification.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanConst.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanLinear.cs" />
+    <Compile Include="GaussianProcess\MeanFunctions\MeanProduct.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\MeanFunctions\MeanSum.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\MeanFunctions\MeanZero.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessHyperparameterInitializer.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionSolutionCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionModelCreator.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessModelCreator.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="GaussianProcess\Util.cs" />
+    <Compile Include="GaussianProcess\IMeanFunction.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessModel.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegression.cs" />
+    <Compile Include="GaussianProcess\GaussianProcessRegressionSolution.cs" />
+    <Compile Include="GaussianProcess\ICovarianceFunction.cs" />
+    <Compile Include="Interfaces\IGaussianProcessModel.cs" />
+    <Compile Include="Interfaces\IGaussianProcessSolution.cs" />
+    <Compile Include="Interfaces\INcaClassificationSolution.cs" />
+    <Compile Include="Interfaces\INcaModel.cs" />
+    <Compile Include="Interfaces\INearestNeighbourClassificationSolution.cs" />
+    <Compile Include="Interfaces\INearestNeighbourRegressionSolution.cs" />
+    <Compile Include="Interfaces\INearestNeighbourModel.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleClassificationSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleRegressionSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkEnsembleModel.cs" />
+    <Compile Include="Interfaces\INeuralNetworkClassificationSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkRegressionSolution.cs" />
+    <Compile Include="Interfaces\INeuralNetworkModel.cs" />
+    <Compile Include="Interfaces\IRandomForestClassificationSolution.cs" />
+    <Compile Include="Interfaces\IRandomForestModel.cs" />
+    <Compile Include="Interfaces\IRandomForestRegressionSolution.cs" />
+    <Compile Include="Interfaces\ISupportVectorMachineModel.cs" />
+    <Compile Include="Interfaces\ISupportVectorMachineSolution.cs" />
+    <Compile Include="Interfaces\IDataAnalysisAlgorithm.cs" />
+    <Compile Include="kMeans\KMeansClustering.cs" />
+    <Compile Include="kMeans\KMeansClusteringModel.cs" />
+    <Compile Include="kMeans\KMeansClusteringSolution.cs" />
+    <Compile Include="kMeans\KMeansClusteringUtil.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="Linear\AlglibUtil.cs" />
+    <Compile Include="Linear\Scaling.cs" />
+    <Compile Include="Linear\LinearDiscriminantAnalysis.cs" />
+    <Compile Include="Linear\LinearRegression.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="Linear\MultinomialLogitClassification.cs" />
+    <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
+    <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="Nca\Initialization\INcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\LdaInitializer.cs" />
+    <Compile Include="Nca\Initialization\PcaInitializer.cs" />
+    <Compile Include="Nca\Initialization\RandomInitializer.cs" />
+    <Compile Include="Nca\Matrix.cs" />
+    <Compile Include="Nca\NcaAlgorithm.cs" />
+    <Compile Include="Nca\NcaClassificationSolution.cs" />
+    <Compile Include="Nca\NcaModel.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourClassification.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourClassificationSolution.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourModel.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourRegression.cs" />
+    <Compile Include="NearestNeighbour\NearestNeighbourRegressionSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleClassification.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleClassificationSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleRegressionSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkEnsembleRegression.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassification.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassificationSolution.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkRegression.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkRegressionSolution.cs" />
+    <Compile Include="Plugin.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RandomForest\RandomForestClassificationSolution.cs" />
+    <Compile Include="RandomForest\RandomForestClassification.cs" />
+    <Compile Include="RandomForest\RandomForestModel.cs" />
+    <Compile Include="RandomForest\RandomForestRegression.cs" />
+    <Compile Include="RandomForest\RandomForestRegressionSolution.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorClassification.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorClassificationSolution.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorMachineModel.cs" />
+    <Compile Include="SupportVectorMachine\SupportVectorMachineUtil.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="SupportVectorMachine\SupportVectorRegression.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="SupportVectorMachine\SupportVectorRegressionSolution.cs">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="TimeSeries\AutoregressiveModeling.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.GradientDescent\3.3\HeuristicLab.Algorithms.GradientDescent.csproj">
+      <Project>{1256B945-EEA9-4BE4-9880-76B5B113F089}</Project>
+      <Name>HeuristicLab.Algorithms.GradientDescent</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
+      <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
+      <Name>HeuristicLab.Analysis-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
+      <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>
+      <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common.Resources\3.3\HeuristicLab.Common.Resources-3.3.csproj">
+      <Project>{0E27A536-1C4A-4624-A65E-DC4F4F23E3E1}</Project>
+      <Name>HeuristicLab.Common.Resources-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
+      <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
+      <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
+      <Project>{C36BD924-A541-4A00-AFA8-41701378DDC5}</Project>
+      <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
+      <Project>{BBAB9DF5-5EF3-4BA8-ADE9-B36E82114937}</Project>
+      <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.RealVectorEncoding\3.3\HeuristicLab.Encodings.RealVectorEncoding-3.3.csproj">
+      <Project>{BB6D334A-4BB6-4674-9883-31A6EBB32CAB}</Project>
+      <Name>HeuristicLab.Encodings.RealVectorEncoding-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
+      <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
+      <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Operators\3.3\HeuristicLab.Operators-3.3.csproj">
+      <Project>{23DA7FF4-D5B8-41B6-AA96-F0561D24F3EE}</Project>
+      <Name>HeuristicLab.Operators-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
+      <Project>{14AB8D24-25BC-400C-A846-4627AA945192}</Project>
+      <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
+      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
+      <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
+      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
+      <Name>HeuristicLab.Persistence-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4.csproj">
+      <Project>{05BAE4E1-A9FA-4644-AA77-42558720159E}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Classification-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj">
+      <Project>{5AC82412-911B-4FA2-A013-EDC5E3F3FCC2}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4.csproj">
+      <Project>{07486E68-1517-4B9D-A58D-A38E99AE71AB}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis-3.4</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis.Symbolic\3.4\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj">
+      <Project>{3D28463F-EC96-4D82-AFEE-38BE91A0CA00}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis.Symbolic-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.DataAnalysis\3.4\HeuristicLab.Problems.DataAnalysis-3.4.csproj">
+      <Project>{DF87C13E-A889-46FF-8153-66DCAA8C5674}</Project>
+      <Name>HeuristicLab.Problems.DataAnalysis-3.4</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
+      <Project>{3540E29E-4793-49E7-8EE2-FEA7F61C3994}</Project>
+      <Name>HeuristicLab.Problems.Instances-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="HeuristicLab.snk" />
+    <None Include="Plugin.cs.frame" />
+    <None Include="Properties\AssemblyInfo.cs.frame" />
+  </ItemGroup>
+  <ItemGroup>
+    <BootstrapperPackage Include="Microsoft.Net.Client.3.5">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
+      <Install>false</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
+      <Visible>False</Visible>
+      <ProductName>Windows Installer 3.1</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+  <PropertyGroup>
+    <PreBuildEvent Condition=" '$(OS)' == 'Windows_NT' ">set Path=%25Path%25;$(ProjectDir);$(SolutionDir)
+set ProjectDir=$(ProjectDir)
+set SolutionDir=$(SolutionDir)
+set Outdir=$(Outdir)
+
+call PreBuildEvent.cmd
+</PreBuildEvent>
+    <PreBuildEvent Condition=" '$(OS)' != 'Windows_NT' ">
+export ProjectDir=$(ProjectDir)
+export SolutionDir=$(SolutionDir)
+
+$SolutionDir/PreBuildEvent.sh
+</PreBuildEvent>
+  </PropertyGroup>
+</Project>
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IDataAnalysisAlgorithm.cs	(revision 9070)
@@ -0,0 +1,32 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a data analysis algorithm.
+  /// </summary>
+  public interface IDataAnalysisAlgorithm<T> : IAlgorithm where T : class, IDataAnalysisProblem {
+    new T Problem { get; set; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessModel.cs	(revision 9070)
@@ -0,0 +1,39 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a Gaussian process posterior
+  /// </summary>
+  public interface IGaussianProcessModel : IRegressionModel {
+    double NegativeLogLikelihood { get; }
+    double SigmaNoise { get; }
+    IMeanFunction MeanFunction { get; }
+    ICovarianceFunction CovarianceFunction { get; }
+    double[] HyperparameterGradients { get; }
+
+    IEnumerable<double> GetEstimatedVariance(Dataset ds, IEnumerable<int> rows);
+    void FixParameters();
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IGaussianProcessSolution.cs	(revision 9070)
@@ -0,0 +1,31 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a Gaussian process solution (either regression or classification)
+  /// </summary>
+  public interface IGaussianProcessSolution : IDataAnalysisSolution {
+    new IGaussianProcessModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,28 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaClassificationSolution : IClassificationSolution {
+    new INcaModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INcaModel.cs	(revision 9070)
@@ -0,0 +1,31 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaModel : IClassificationModel {
+    new INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData);
+
+    double[,] Reduce(Dataset dataset, IEnumerable<int> rows);
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a nearest neighbour classification solution
+  /// </summary>
+  public interface INearestNeighbourClassificationSolution : IClassificationSolution {
+    new INearestNeighbourModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourModel.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a nearest neighbour model for either regression or classification
+  /// </summary>
+  public interface INearestNeighbourModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INearestNeighbourRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a nearest neighbour regression solution
+  /// </summary>
+  public interface INearestNeighbourRegressionSolution : IRegressionSolution {
+    new INearestNeighbourModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network classification solution
+  /// </summary>
+  public interface INeuralNetworkClassificationSolution : IClassificationSolution {
+    new INeuralNetworkModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network ensemble classification solution
+  /// </summary>
+  public interface INeuralNetworkEnsembleClassificationSolution : IClassificationSolution {
+    new INeuralNetworkEnsembleModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleModel.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network ensemble model for either regression or classification
+  /// </summary>
+  public interface INeuralNetworkEnsembleModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkEnsembleRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network ensemble regression solution
+  /// </summary>
+  public interface INeuralNetworkEnsembleRegressionSolution : IRegressionSolution {
+    new INeuralNetworkEnsembleModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkModel.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network model for either regression or classification
+  /// </summary>
+  public interface INeuralNetworkModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a neural network regression solution
+  /// </summary>
+  public interface INeuralNetworkRegressionSolution : IRegressionSolution {
+    new INeuralNetworkModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a random forest classification solution
+  /// </summary>
+  public interface IRandomForestClassificationSolution : IClassificationSolution {
+    new IRandomForestModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestModel.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a random forest model for either regression or classification
+  /// </summary>
+  public interface IRandomForestModel : IRegressionModel, IClassificationModel {
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/IRandomForestRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a random forest regression solution
+  /// </summary>
+  public interface IRandomForestRegressionSolution : IRegressionSolution {
+    new IRandomForestModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineModel.cs	(revision 9070)
@@ -0,0 +1,34 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a support vector machine model
+  /// </summary>
+  public interface ISupportVectorMachineModel : IDataAnalysisModel, IRegressionModel, IClassificationModel {
+    svm_model Model { get; }
+    RangeTransform RangeTransform { get; }
+    Dataset SupportVectors { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/ISupportVectorMachineSolution.cs	(revision 9070)
@@ -0,0 +1,33 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Optimization;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Interface to represent a support vector solution (either regression or classification)
+  /// </summary>
+  public interface ISupportVectorMachineSolution : IDataAnalysisSolution {
+    new ISupportVectorMachineModel Model { get; }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs	(revision 9070)
@@ -0,0 +1,67 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class AlglibUtil {
+    public static double[,] PrepareInputMatrix(Dataset dataset, IEnumerable<string> variables, IEnumerable<int> rows) {
+      List<string> variablesList = variables.ToList();
+      List<int> rowsList = rows.ToList();
+
+      double[,] matrix = new double[rowsList.Count, variablesList.Count];
+
+      int col = 0;
+      foreach (string column in variables) {
+        var values = dataset.GetDoubleValues(column, rows);
+        int row = 0;
+        foreach (var value in values) {
+          matrix[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+
+      return matrix;
+    }
+    public static double[,] PrepareAndScaleInputMatrix(Dataset dataset, IEnumerable<string> variables, IEnumerable<int> rows, Scaling scaling) {
+      List<string> variablesList = variables.ToList();
+      List<int> rowsList = rows.ToList();
+
+      double[,] matrix = new double[rowsList.Count, variablesList.Count];
+
+      int col = 0;
+      foreach (string column in variables) {
+        var values = scaling.GetScaledValues(dataset, column, rows);
+        int row = 0;
+        foreach (var value in values) {
+          matrix[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+
+      return matrix;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs	(revision 9070)
@@ -0,0 +1,118 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Classification;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Linear discriminant analysis classification algorithm.
+  /// </summary>
+  [Item("Linear Discriminant Analysis", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class LinearDiscriminantAnalysis : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string LinearDiscriminantAnalysisSolutionResultName = "Linear discriminant analysis solution";
+
+    [StorableConstructor]
+    private LinearDiscriminantAnalysis(bool deserializing) : base(deserializing) { }
+    private LinearDiscriminantAnalysis(LinearDiscriminantAnalysis original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public LinearDiscriminantAnalysis()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearDiscriminantAnalysis(this, cloner);
+    }
+
+    #region Fisher LDA
+    protected override void Run() {
+      var solution = CreateLinearDiscriminantAnalysisSolution(Problem.ProblemData);
+      Results.Add(new Result(LinearDiscriminantAnalysisSolutionResultName, "The linear discriminant analysis.", solution));
+    }
+
+    public static IClassificationSolution CreateLinearDiscriminantAnalysisSolution(IClassificationProblemData problemData) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      int nClasses = problemData.ClassNames.Count();
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Linear discriminant analysis does not support NaN or infinity values in the input dataset.");
+
+      // change class values into class index
+      int targetVariableColumn = inputMatrix.GetLength(1) - 1;
+      List<double> classValues = problemData.ClassValues.OrderBy(x => x).ToList();
+      for (int row = 0; row < inputMatrix.GetLength(0); row++) {
+        inputMatrix[row, targetVariableColumn] = classValues.IndexOf(inputMatrix[row, targetVariableColumn]);
+      }
+      int info;
+      double[] w;
+      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), allowedInputVariables.Count(), nClasses, out info, out w);
+      if (info < 1) throw new ArgumentException("Error in calculation of linear discriminant analysis solution");
+
+      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
+      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
+      tree.Root.AddSubtree(startNode);
+      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
+      startNode.AddSubtree(addition);
+
+      int col = 0;
+      foreach (string column in allowedInputVariables) {
+        VariableTreeNode vNode = (VariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
+        vNode.VariableName = column;
+        vNode.Weight = w[col];
+        addition.AddSubtree(vNode);
+        col++;
+      }
+
+      var model = LinearDiscriminantAnalysis.CreateDiscriminantFunctionModel(tree, new SymbolicDataAnalysisExpressionTreeInterpreter(), problemData, rows);
+      SymbolicDiscriminantFunctionClassificationSolution solution = new SymbolicDiscriminantFunctionClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+
+      return solution;
+    }
+    #endregion
+
+    private static SymbolicDiscriminantFunctionClassificationModel CreateDiscriminantFunctionModel(ISymbolicExpressionTree tree,
+      ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
+      IClassificationProblemData problemData,
+      IEnumerable<int> rows) {
+      var model = new SymbolicDiscriminantFunctionClassificationModel(tree, interpreter, new AccuracyMaximizationThresholdCalculator());
+      model.RecalculateModelParameters(problemData, rows);
+      return model;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs	(revision 9070)
@@ -0,0 +1,119 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Linear regression data analysis algorithm.
+  /// </summary>
+  [Item("Linear Regression", "Linear regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class LinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string LinearRegressionModelResultName = "Linear regression solution";
+
+    [StorableConstructor]
+    private LinearRegression(bool deserializing) : base(deserializing) { }
+    private LinearRegression(LinearRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public LinearRegression()
+      : base() {
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LinearRegression(this, cloner);
+    }
+
+    #region linear regression
+    protected override void Run() {
+      double rmsError, cvRmsError;
+      var solution = CreateLinearRegressionSolution(Problem.ProblemData, out rmsError, out cvRmsError);
+      Results.Add(new Result(LinearRegressionModelResultName, "The linear regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the linear regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Estimated root mean square error (cross-validation)", "The estimated root of the mean of squared errors of the linear regression solution via cross validation.", new DoubleValue(cvRmsError)));
+    }
+
+    public static ISymbolicRegressionSolution CreateLinearRegressionSolution(IRegressionProblemData problemData, out double rmsError, out double cvRmsError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Linear regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
+
+      int retVal = 1;
+      alglib.lrbuild(inputMatrix, nRows, nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmsError = ar.rmserror;
+      cvRmsError = ar.cvrmserror;
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
+      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
+      tree.Root.AddSubtree(startNode);
+      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
+      startNode.AddSubtree(addition);
+
+      int col = 0;
+      foreach (string column in allowedInputVariables) {
+        VariableTreeNode vNode = (VariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
+        vNode.VariableName = column;
+        vNode.Weight = coefficients[col];
+        addition.AddSubtree(vNode);
+        col++;
+      }
+
+      ConstantTreeNode cNode = (ConstantTreeNode)new Constant().CreateTreeNode();
+      cNode.Value = coefficients[coefficients.Length - 1];
+      addition.AddSubtree(cNode);
+
+      SymbolicRegressionSolution solution = new SymbolicRegressionSolution(new SymbolicRegressionModel(tree, new SymbolicDataAnalysisExpressionTreeInterpreter()), (IRegressionProblemData)problemData.Clone());
+      solution.Model.Name = "Linear Regression Model";
+      solution.Name = "Linear Regression Solution";
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs	(revision 9070)
@@ -0,0 +1,102 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Multinomial logit regression data analysis algorithm.
+  /// </summary>
+  [Item("Multinomial Logit Classification", "Multinomial logit classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class MultiNomialLogitClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string LogitClassificationModelResultName = "Logit classification solution";
+
+    [StorableConstructor]
+    private MultiNomialLogitClassification(bool deserializing) : base(deserializing) { }
+    private MultiNomialLogitClassification(MultiNomialLogitClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MultiNomialLogitClassification()
+      : base() {
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultiNomialLogitClassification(this, cloner);
+    }
+
+    #region logit classification
+    protected override void Run() {
+      double rmsError, relClassError;
+      var solution = CreateLogitClassificationSolution(Problem.ProblemData, out rmsError, out relClassError);
+      Results.Add(new Result(LogitClassificationModelResultName, "The logit classification solution.", solution));
+      Results.Add(new Result("Root mean squared error", "The root of the mean of squared errors of the logit regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Relative classification error", "Relative classification error on the training set (percentage of misclassified cases).", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateLogitClassificationSolution(IClassificationProblemData problemData, out double rmsError, out double relClassError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Multinomial logit classification does not support NaN or infinity values in the input dataset.");
+
+      alglib.logitmodel lm = new alglib.logitmodel();
+      alglib.mnlreport rep = new alglib.mnlreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+      int info;
+      alglib.mnltrainh(inputMatrix, nRows, nFeatures, nClasses, out info, out lm, out rep);
+      if (info != 1) throw new ArgumentException("Error in calculation of logit classification solution");
+
+      rmsError = alglib.mnlrmserror(lm, inputMatrix, nRows);
+      relClassError = alglib.mnlrelclserror(lm, inputMatrix, nRows);
+
+      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution((IClassificationProblemData)problemData.Clone(), new MultinomialLogitModel(lm, targetVariable, allowedInputVariables, classValues));
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a multinomial logit solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("Multinomial Logit Classification Solution", "Represents a multinomial logit solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class MultinomialLogitClassificationSolution : ClassificationSolution {
+
+    public new MultinomialLogitModel Model {
+      get { return (MultinomialLogitModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private MultinomialLogitClassificationSolution(bool deserializing) : base(deserializing) { }
+    private MultinomialLogitClassificationSolution(MultinomialLogitClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public MultinomialLogitClassificationSolution(IClassificationProblemData problemData, MultinomialLogitModel logitModel)
+      : base(logitModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultinomialLogitClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs	(revision 9070)
@@ -0,0 +1,138 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a multinomial logit model for classification
+  /// </summary>
+  [StorableClass]
+  [Item("Multinomial Logit Model", "Represents a multinomial logit model for classification.")]
+  public sealed class MultinomialLogitModel : NamedItem, IClassificationModel {
+
+    private alglib.logitmodel logitModel;
+    public alglib.logitmodel Model {
+      get { return logitModel; }
+      set {
+        if (value != logitModel) {
+          if (value == null) throw new ArgumentNullException();
+          logitModel = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private MultinomialLogitModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        logitModel = new alglib.logitmodel();
+    }
+    private MultinomialLogitModel(MultinomialLogitModel original, Cloner cloner)
+      : base(original, cloner) {
+      logitModel = new alglib.logitmodel();
+      logitModel.innerobj.w = (double[])original.logitModel.innerobj.w.Clone();
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      classValues = (double[])original.classValues.Clone();
+    }
+    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.logitModel = logitModel;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new MultinomialLogitModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mnlprocess(logitModel, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public MultinomialLogitClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new MultinomialLogitClassificationSolution(new ClassificationProblemData(problemData), this);
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private double[] LogitModelW {
+      get {
+        return logitModel.innerobj.w;
+      }
+      set {
+        logitModel.innerobj.w = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/Scaling.cs	(revision 9070)
@@ -0,0 +1,68 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [StorableClass]
+  [Item(Name = "Scaling", Description = "Contains information about scaling of variables for data-analysis algorithms.")]
+  public class Scaling : Item {
+    [Storable]
+    private Dictionary<string, Tuple<double, double>> scalingParameters = new Dictionary<string, Tuple<double, double>>();
+    [StorableConstructor]
+    protected Scaling(bool deserializing) : base(deserializing) { }
+    protected Scaling(Scaling original, Cloner cloner)
+      : base(original, cloner) {
+      foreach (var pair in original.scalingParameters)
+        scalingParameters.Add(pair.Key, Tuple.Create(pair.Value.Item1, pair.Value.Item2));
+    }
+    public Scaling(Dataset ds, IEnumerable<string> variables, IEnumerable<int> rows) {
+      foreach (var variable in variables) {
+        var values = ds.GetDoubleValues(variable, rows);
+        var min = values.Where(x => !double.IsNaN(x)).Min();
+        var max = values.Where(x => !double.IsNaN(x)).Max();
+        scalingParameters[variable] = Tuple.Create(min, max);
+      }
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new Scaling(this, cloner);
+    }
+
+    public IEnumerable<double> GetScaledValues(Dataset ds, string variable, IEnumerable<int> rows) {
+      double min = scalingParameters[variable].Item1;
+      double max = scalingParameters[variable].Item2;
+      if (min.IsAlmost(max)) return rows.Select(i => 0.0); // return enumerable of zeros
+      return ds.GetDoubleValues(variable, rows).Select(x => (x - min) / (max - min));  // scale to range [0..1]
+    }
+
+    public void GetScalingParameters(string variable, out double min, out double max) {
+      min = scalingParameters[variable].Item1;
+      max = scalingParameters[variable].Item2;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 9070)
@@ -0,0 +1,35 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INCAInitializer : IItem {
+    /// <summary>
+    /// Calculates an initial projection for the NCA to start from.
+    /// </summary>
+    /// <param name="data">The problem data that contains the AllowedInputVariables and TrainingIndices.</param>
+    /// <param name="dimensions">The amount of columns in the matrix</param>
+    /// <returns>A flat representation of a matrix that is read row-wise and contains AllowedInputVariables * TrainingIndices numbers.</returns>
+    double[] Initialize(IClassificationProblemData data, int dimensions);
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 9070)
@@ -0,0 +1,80 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("LDA", "Initializes the matrix by performing a linear discriminant analysis.")]
+  [StorableClass]
+  public class LDAInitializer : Item, INCAInitializer {
+
+    [StorableConstructor]
+    protected LDAInitializer(bool deserializing) : base(deserializing) { }
+    protected LDAInitializer(LDAInitializer original, Cloner cloner) : base(original, cloner) { }
+    public LDAInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new LDAInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var ldaDs = new double[instances, attributes + 1];
+      int row, col = 0;
+      foreach (var variable in data.AllowedInputVariables) {
+        row = 0;
+        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
+          ldaDs[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+      row = 0;
+      var uniqueClasses = new Dictionary<double, int>();
+      foreach (var label in data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices)) {
+        if (!uniqueClasses.ContainsKey(label))
+          uniqueClasses[label] = uniqueClasses.Count;
+        ldaDs[row++, attributes] = label;
+      }
+      for (row = 0; row < instances; row++)
+        ldaDs[row, attributes] = uniqueClasses[ldaDs[row, attributes]];
+
+      int info;
+      double[,] matrix;
+      alglib.fisherldan(ldaDs, instances, attributes, uniqueClasses.Count, out info, out matrix);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      return result;
+    }
+
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 9070)
@@ -0,0 +1,71 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("PCA", "Initializes the matrix by performing a principal components analysis.")]
+  [StorableClass]
+  public sealed class PCAInitializer : Item, INCAInitializer {
+
+    [StorableConstructor]
+    private PCAInitializer(bool deserializing) : base(deserializing) { }
+    private PCAInitializer(PCAInitializer original, Cloner cloner) : base(original, cloner) { }
+    public PCAInitializer() : base() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PCAInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var pcaDs = new double[instances, attributes];
+      int col = 0;
+      foreach (var variable in data.AllowedInputVariables) {
+        int row = 0;
+        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
+          pcaDs[row, col] = value;
+          row++;
+        }
+        col++;
+      }
+
+      int info;
+      double[] varianceValues;
+      double[,] matrix;
+      alglib.pcabuildbasis(pcaDs, instances, attributes, out info, out varianceValues, out matrix);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      return result;
+    }
+
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 9070)
@@ -0,0 +1,81 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("Random", "Initializes the matrix randomly.")]
+  [StorableClass]
+  public class RandomInitializer : ParameterizedNamedItem, INCAInitializer {
+    private IValueParameter<IntValue> RandomParameter {
+      get { return (IValueParameter<IntValue>)Parameters["Seed"]; }
+    }
+    private IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters["SetSeedRandomly"]; }
+    }
+
+    public int Seed {
+      get { return RandomParameter.Value.Value; }
+      set { RandomParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected RandomInitializer(bool deserializing) : base(deserializing) { }
+    protected RandomInitializer(RandomInitializer original, Cloner cloner) : base(original, cloner) { }
+    public RandomInitializer()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>("Seed", "The seed for the random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>("SetSeedRandomly", "Whether the seed should be randomized for each call.", new BoolValue(true)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomInitializer(this, cloner);
+    }
+
+    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+      var instances = data.TrainingIndices.Count();
+      var attributes = data.AllowedInputVariables.Count();
+
+      var random = new MersenneTwister();
+      if (SetSeedRandomly) Seed = random.Next();
+      random.Reset(Seed);
+
+      var range = data.AllowedInputVariables.Select(x => data.Dataset.GetDoubleValues(x).Max() - data.Dataset.GetDoubleValues(x).Min()).ToArray();
+      var matrix = new double[attributes * dimensions];
+      for (int i = 0; i < matrix.Length; i++)
+        matrix[i] = random.NextDouble() / range[i / dimensions];
+
+      return matrix;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Matrix.cs	(revision 9070)
@@ -0,0 +1,228 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [NonDiscoverableType]
+  internal class Matrix : IEnumerable<double>, IDeepCloneable {
+    // this type is immutable
+    private readonly IEnumerable<double> values;
+    public readonly int Rows;
+    public readonly int Columns;
+
+    protected Matrix(Matrix original, Cloner cloner) {
+      this.values = original.values.ToArray();
+      this.Rows = original.Rows;
+      this.Columns = original.Columns;
+      cloner.RegisterClonedObject(original, this);
+    }
+    public Matrix(IEnumerable<double> vector) {
+      this.values = vector;
+      Rows = 1;
+      Columns = vector.Count();
+    }
+    public Matrix(IEnumerable<double> vector, int length) {
+      this.values = vector;
+      Rows = 1;
+      Columns = length;
+    }
+    public Matrix(double[,] matrix) {
+      this.values = GetOnlineValues(matrix);
+      Rows = matrix.GetLength(0);
+      Columns = matrix.GetLength(1);
+    }
+    public Matrix(IEnumerable<double> matrix, int rows, int columns) {
+      this.values = matrix;
+      Rows = rows;
+      Columns = columns;
+    }
+
+    public object Clone() {
+      return Clone(new Cloner());
+    }
+    public IDeepCloneable Clone(Cloner cloner) {
+      return new Matrix(this, cloner);
+    }
+
+    public Matrix Transpose() {
+      return new Matrix(Transpose(values, Columns, Rows), Columns, Rows);
+    }
+
+    private IEnumerable<double> Transpose(IEnumerable<double> values, int rows, int columns) {
+      // vectors don't need to be transposed
+      if (rows == 1 || columns == 1) {
+        foreach (var v in values) yield return v;
+        yield break;
+      }
+
+      int skip = 0;
+      var iter = values.GetEnumerator();
+      if (!iter.MoveNext()) yield break;
+      while (skip < rows) {
+        for (int i = 0; i < skip; i++) iter.MoveNext();
+        yield return iter.Current;
+        for (int j = 0; j < columns - 1; j++) {
+          for (int i = 0; i < rows; i++) iter.MoveNext();
+          yield return iter.Current;
+        }
+        skip++;
+        if (skip < rows) {
+          iter = values.GetEnumerator();
+          iter.MoveNext();
+        }
+      }
+    }
+
+    public Matrix Add(Matrix other) {
+      return new Matrix(AddOnline(other), Rows, Columns);
+    }
+
+    public void AddTo(double[,] matrix) {
+      if (Rows != matrix.GetLength(0) || Columns != matrix.GetLength(1)) throw new ArgumentException("unequal size", "matrix");
+      var iter = values.GetEnumerator();
+      for (int i = 0; i < Rows; i++)
+        for (int j = 0; j < Columns; j++) {
+          iter.MoveNext();
+          matrix[i, j] += iter.Current;
+        }
+    }
+
+    public Matrix Subtract(Matrix other) {
+      return new Matrix(SubtractOnline(other), Rows, Columns);
+    }
+
+    public Matrix Multiply(Matrix other) {
+      return new Matrix(MultiplyOnline(other), Rows, other.Columns);
+    }
+
+    public Matrix Multiply(double value) {
+      return new Matrix(values.Select(x => x * value), Rows, Columns);
+    }
+
+    public double EuclideanNorm() {
+      return Math.Sqrt(SumOfSquares());
+    }
+
+    public double SumOfSquares() {
+      return values.Sum(x => x * x);
+    }
+
+    public Matrix OuterProduct(Matrix other) {
+      if (Rows != 1 || other.Rows != 1) throw new ArgumentException("OuterProduct can only be applied to vectors.");
+      return Transpose().Multiply(other);
+    }
+
+    public IEnumerable<double> ColumnSums() {
+      return Transpose().RowSums();
+    }
+
+    public IEnumerable<double> RowSums() {
+      var sum = 0.0;
+      int counter = 0;
+      foreach (var v in values) {
+        sum += v;
+        counter++;
+        if (counter == Rows) {
+          yield return sum;
+          sum = 0.0;
+          counter = 0;
+        }
+      }
+    }
+
+    public Matrix Negate() {
+      return new Matrix(values.Select(x => -x), Rows, Columns);
+    }
+
+    public Matrix Apply() {
+      return new Matrix(values.ToArray(), Rows, Columns);
+    }
+
+    public IEnumerator<double> GetEnumerator() { return values.GetEnumerator(); }
+    IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
+
+
+    private IEnumerable<double> AddOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current + otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> SubtractOnline(Matrix other) {
+      if (Rows != other.Rows || Columns != other.Columns) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherIter = other.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int i = 0; i < Rows * Columns; i++) {
+        yield return meIter.Current - otherIter.Current;
+        meIter.MoveNext();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> MultiplyOnline(Matrix other) {
+      if (Columns != other.Rows) throw new ArgumentException("Number of rows and columns are not equal.");
+      var meIter = values.GetEnumerator();
+      var otherByColumn = other.Transpose();
+      var otherIter = otherByColumn.GetEnumerator();
+      if (!meIter.MoveNext()) yield break;
+      if (!otherIter.MoveNext()) yield break;
+      for (int r = 0; r < Rows; r++) {
+        var row = new double[Columns];
+        for (int x = 0; x < Columns; x++) {
+          row[x] = meIter.Current;
+          meIter.MoveNext();
+        }
+        for (int c = 0; c < other.Columns; c++) {
+          var sum = 0.0;
+          for (int y = 0; y < other.Rows; y++) {
+            sum += row[y] * otherIter.Current;
+            otherIter.MoveNext();
+          }
+          yield return sum;
+        }
+        otherIter = otherByColumn.GetEnumerator();
+        otherIter.MoveNext();
+      }
+    }
+
+    private IEnumerable<double> GetOnlineValues(double[,] matrix) {
+      for (int i = 0; i < matrix.GetLength(0); i++)
+        for (int j = 0; j < matrix.GetLength(1); j++) {
+          yield return matrix[i, j];
+        }
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 9070)
@@ -0,0 +1,308 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  internal delegate void Reporter(double quality, double[] coefficients, double[] gradients);
+  /// <summary>
+  /// Neighborhood Components Analysis
+  /// </summary>
+  [Item("Neighborhood Components Analysis (NCA)", @"Implementation of Neighborhood Components Analysis
+based on the description of J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov. 2005.
+Neighbourhood Component Analysis. Advances in Neural Information Processing Systems, 17. pp. 513-520
+with additional regularizations described in Z. Yang, J. Laaksonen. 2007.
+Regularized Neighborhood Component Analysis. Lecture Notes in Computer Science, 4522. pp. 253-262.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NcaAlgorithm : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    #region Parameter Properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["K"]; }
+    }
+    public IFixedValueParameter<IntValue> DimensionsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+    public IConstrainedValueParameter<INCAInitializer> InitializationParameter {
+      get { return (IConstrainedValueParameter<INCAInitializer>)Parameters["Initialization"]; }
+    }
+    public IFixedValueParameter<IntValue> NeighborSamplesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["NeighborSamples"]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters["Iterations"]; }
+    }
+    public IFixedValueParameter<DoubleValue> RegularizationParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["Regularization"]; }
+    }
+    #endregion
+
+    #region Properties
+    public int K {
+      get { return KParameter.Value.Value; }
+      set { KParameter.Value.Value = value; }
+    }
+    public int Dimensions {
+      get { return DimensionsParameter.Value.Value; }
+      set { DimensionsParameter.Value.Value = value; }
+    }
+    public int NeighborSamples {
+      get { return NeighborSamplesParameter.Value.Value; }
+      set { NeighborSamplesParameter.Value.Value = value; }
+    }
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+    public double Regularization {
+      get { return RegularizationParameter.Value.Value; }
+      set { RegularizationParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NcaAlgorithm(bool deserializing) : base(deserializing) { }
+    private NcaAlgorithm(NcaAlgorithm original, Cloner cloner) : base(original, cloner) { }
+    public NcaAlgorithm()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>("K", "The K for the nearest neighbor.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Dimensions", "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
+      Parameters.Add(new ConstrainedValueParameter<INCAInitializer>("Initialization", "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
+      Parameters.Add(new FixedValueParameter<IntValue>("NeighborSamples", "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one.", new IntValue(60)));
+      Parameters.Add(new FixedValueParameter<IntValue>("Iterations", "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
+
+      INCAInitializer defaultInitializer = null;
+      foreach (var initializer in ApplicationManager.Manager.GetInstances<INCAInitializer>().OrderBy(x => x.ItemName)) {
+        if (initializer is LDAInitializer) defaultInitializer = initializer;
+        InitializationParameter.ValidValues.Add(initializer);
+      }
+      if (defaultInitializer != null) InitializationParameter.Value = defaultInitializer;
+
+      Problem = new ClassificationProblem();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaAlgorithm(this, cloner);
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey("Regularization")) {
+        Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
+      }
+    }
+
+    public override void Prepare() {
+      if (Problem != null) base.Prepare();
+    }
+
+    protected override void Run() {
+      var initializer = InitializationParameter.Value;
+
+      var clonedProblem = (IClassificationProblemData)Problem.ProblemData.Clone();
+      var model = Train(clonedProblem, K, Dimensions, NeighborSamples, Regularization, Iterations, initializer.Initialize(clonedProblem, Dimensions), ReportQuality, CancellationToken.None);
+      var solution = model.CreateClassificationSolution(clonedProblem);
+      if (!Results.ContainsKey("ClassificationSolution"))
+        Results.Add(new Result("ClassificationSolution", "The classification solution.", solution));
+      else Results["ClassificationSolution"].Value = solution;
+    }
+
+    public static INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
+      var clonedProblem = (IClassificationProblemData)data.Clone();
+      var model = Train(clonedProblem, k, dimensions, neighborSamples, regularization, iterations, initializer);
+      return model.CreateClassificationSolution(clonedProblem);
+    }
+
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
+      return Train(problemData, k, dimensions, neighborSamples, regularization, iterations, initializer.Initialize(problemData, dimensions), null, CancellationToken.None);
+    }
+
+    public static INcaModel Train(IClassificationProblemData problemData, int k, int neighborSamples, double regularization, int iterations, double[,] initalMatrix) {
+      var matrix = new double[initalMatrix.Length];
+      for (int i = 0; i < initalMatrix.GetLength(0); i++)
+        for (int j = 0; j < initalMatrix.GetLength(1); j++)
+          matrix[i * initalMatrix.GetLength(1) + j] = initalMatrix[i, j];
+      return Train(problemData, k, initalMatrix.GetLength(1), neighborSamples, regularization, iterations, matrix, null, CancellationToken.None);
+    }
+
+    private static INcaModel Train(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, double[] matrix, Reporter reporter, CancellationToken cancellation) {
+      var scaling = new Scaling(data.Dataset, data.AllowedInputVariables, data.TrainingIndices);
+      var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices, scaling);
+      var classes = data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices).ToArray();
+      var attributes = scaledData.GetLength(1);
+
+      alglib.mincgstate state;
+      alglib.mincgreport rep;
+      alglib.mincgcreate(matrix, out state);
+      alglib.mincgsetcond(state, 0, 0, 0, iterations);
+      alglib.mincgsetxrep(state, true);
+      //alglib.mincgsetgradientcheck(state, 0.01);
+      int neighborSampleSize = neighborSamples;
+      Optimize(state, scaledData, classes, dimensions, neighborSampleSize, regularization, cancellation, reporter);
+      alglib.mincgresults(state, out matrix, out rep);
+      if (rep.terminationtype == -7) throw new InvalidOperationException("Gradient verification failed.");
+
+      var transformationMatrix = new double[attributes, dimensions];
+      var counter = 0;
+      for (var i = 0; i < attributes; i++)
+        for (var j = 0; j < dimensions; j++)
+          transformationMatrix[i, j] = matrix[counter++];
+
+      return new NcaModel(k, transformationMatrix, data.Dataset, data.TrainingIndices, data.TargetVariable, data.AllowedInputVariables, scaling, data.ClassValues.ToArray());
+    }
+
+    private static void Optimize(alglib.mincgstate state, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda, CancellationToken cancellation, Reporter reporter) {
+      while (alglib.mincgiteration(state)) {
+        if (cancellation.IsCancellationRequested) break;
+        if (state.needfg) {
+          Gradient(state.x, ref state.innerobj.f, state.innerobj.g, data, classes, dimensions, neighborSampleSize, lambda);
+          continue;
+        }
+        if (state.innerobj.xupdated) {
+          if (reporter != null)
+            reporter(state.innerobj.f, state.innerobj.x, state.innerobj.g);
+          continue;
+        }
+        throw new InvalidOperationException("Neighborhood Components Analysis: Error in Optimize() (some derivatives were not provided?)");
+      }
+    }
+
+    private static void Gradient(double[] A, ref double func, double[] grad, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda) {
+      var instances = data.GetLength(0);
+      var attributes = data.GetLength(1);
+
+      var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
+
+      alglib.sparsematrix probabilities;
+      alglib.sparsecreate(instances, instances, out probabilities);
+      var transformedDistances = new Dictionary<int, double>(instances);
+      for (int i = 0; i < instances; i++) {
+        var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
+        for (int k = 0; k < instances; k++) {
+          if (k == i) {
+            transformedDistances.Remove(k);
+            continue;
+          }
+          var kVector = new Matrix(GetRow(data, k));
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
+        }
+        var normalization = transformedDistances.Sum(x => x.Value);
+        if (normalization <= 0) continue;
+        foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSampleSize)) {
+          alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+        }
+      }
+      alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
+
+      int t0 = 0, t1 = 0, r, c;
+      double val;
+      var pi = new double[instances];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        if (classes[r].IsAlmost(classes[c])) {
+          pi[r] += val;
+        }
+      }
+
+      var innerSum = new double[attributes, attributes];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
+        vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
+
+        if (classes[r].IsAlmost(classes[c])) {
+          vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
+        }
+      }
+
+      func = -pi.Sum() + lambda * AMatrix.SumOfSquares();
+
+      r = 0;
+      var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
+      foreach (var g in newGrad) {
+        grad[r] = g + lambda * 2 * A[r];
+        r++;
+      }
+    }
+
+    private void ReportQuality(double func, double[] coefficients, double[] gradients) {
+      var instances = Problem.ProblemData.TrainingIndices.Count();
+      DataTable qualities;
+      if (!Results.ContainsKey("Optimization")) {
+        qualities = new DataTable("Optimization");
+        qualities.Rows.Add(new DataRow("Quality", string.Empty));
+        Results.Add(new Result("Optimization", qualities));
+      } else qualities = (DataTable)Results["Optimization"].Value;
+      qualities.Rows["Quality"].Values.Add(-func / instances);
+
+      string[] attributNames = Problem.ProblemData.AllowedInputVariables.ToArray();
+      if (gradients != null) {
+        DataTable grads;
+        if (!Results.ContainsKey("Gradients")) {
+          grads = new DataTable("Gradients");
+          for (int i = 0; i < gradients.Length; i++)
+            grads.Rows.Add(new DataRow(attributNames[i / Dimensions] + "-" + (i % Dimensions), string.Empty));
+          Results.Add(new Result("Gradients", grads));
+        } else grads = (DataTable)Results["Gradients"].Value;
+        for (int i = 0; i < gradients.Length; i++)
+          grads.Rows[attributNames[i / Dimensions] + "-" + (i % Dimensions)].Values.Add(gradients[i]);
+      }
+
+      if (!Results.ContainsKey("Quality")) {
+        Results.Add(new Result("Quality", new DoubleValue(-func / instances)));
+      } else ((DoubleValue)Results["Quality"].Value).Value = -func / instances;
+
+      var attributes = attributNames.Length;
+      var transformationMatrix = new double[attributes, Dimensions];
+      var counter = 0;
+      for (var i = 0; i < attributes; i++)
+        for (var j = 0; j < Dimensions; j++)
+          transformationMatrix[i, j] = coefficients[counter++];
+
+      var scaling = new Scaling(Problem.ProblemData.Dataset, attributNames, Problem.ProblemData.TrainingIndices);
+      var model = new NcaModel(K, transformationMatrix, Problem.ProblemData.Dataset, Problem.ProblemData.TrainingIndices, Problem.ProblemData.TargetVariable, attributNames, scaling, Problem.ProblemData.ClassValues.ToArray());
+
+      IClassificationSolution solution = model.CreateClassificationSolution(Problem.ProblemData);
+      if (!Results.ContainsKey("ClassificationSolution")) {
+        Results.Add(new Result("ClassificationSolution", solution));
+      } else {
+        Results["ClassificationSolution"].Value = solution;
+      }
+    }
+
+    private static IEnumerable<double> GetRow(double[,] data, int row) {
+      for (int i = 0; i < data.GetLength(1); i++)
+        yield return data[row, i];
+    }
+
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,50 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Classification Solution", "")]
+  [StorableClass]
+  public class NcaClassificationSolution : ClassificationSolution, INcaClassificationSolution {
+
+    public new INcaModel Model {
+      get { return (INcaModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    protected NcaClassificationSolution(bool deserializing) : base(deserializing) { }
+    protected NcaClassificationSolution(NcaClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NcaClassificationSolution(IClassificationProblemData problemData, INcaModel ncaModel)
+      : base(ncaModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaModel.cs	(revision 9070)
@@ -0,0 +1,113 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NCA Model", "")]
+  [StorableClass]
+  public class NcaModel : NamedItem, INcaModel {
+
+    [Storable]
+    private Scaling scaling;
+    [Storable]
+    private double[,] transformationMatrix;
+    public double[,] TransformationMatrix {
+      get { return (double[,])transformationMatrix.Clone(); }
+    }
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private INearestNeighbourModel nnModel;
+    [Storable]
+    private double[] classValues;
+
+    [StorableConstructor]
+    protected NcaModel(bool deserializing) : base(deserializing) { }
+    protected NcaModel(NcaModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.scaling = cloner.Clone(original.scaling);
+      this.transformationMatrix = (double[,])original.transformationMatrix.Clone();
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      this.targetVariable = original.targetVariable;
+      this.nnModel = cloner.Clone(original.nnModel);
+      this.classValues = (double[])original.classValues.Clone();
+    }
+    public NcaModel(int k, double[,] transformationMatrix, Dataset dataset, IEnumerable<int> rows, string targetVariable, IEnumerable<string> allowedInputVariables, Scaling scaling, double[] classValues) {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.scaling = scaling;
+      this.transformationMatrix = (double[,])transformationMatrix.Clone();
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.targetVariable = targetVariable;
+      this.classValues = (double[])classValues.Clone();
+
+      var ds = ReduceDataset(dataset, rows);
+      nnModel = new NearestNeighbourModel(ds, Enumerable.Range(0, ds.Rows), k, ds.VariableNames.Last(), ds.VariableNames.Take(transformationMatrix.GetLength(1)), classValues);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      var ds = ReduceDataset(dataset, rows);
+      return nnModel.GetEstimatedClassValues(ds, Enumerable.Range(0, ds.Rows));
+    }
+
+    public INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NcaClassificationSolution(new ClassificationProblemData(problemData), this);
+    }
+
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    public double[,] Reduce(Dataset dataset, IEnumerable<int> rows) {
+      var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(dataset, allowedInputVariables, rows, scaling);
+      var targets = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+      var result = new double[scaledData.GetLength(0), transformationMatrix.GetLength(1) + 1];
+      for (int i = 0; i < scaledData.GetLength(0); i++)
+        for (int j = 0; j < scaledData.GetLength(1); j++) {
+          for (int x = 0; x < transformationMatrix.GetLength(1); x++) {
+            result[i, x] += scaledData[i, j] * transformationMatrix[j, x];
+          }
+          result[i, transformationMatrix.GetLength(1)] = targets[i];
+        }
+      return result;
+    }
+
+    public Dataset ReduceDataset(Dataset dataset, IEnumerable<int> rows) {
+      return new Dataset(Enumerable
+          .Range(0, transformationMatrix.GetLength(1))
+          .Select(x => "X" + x.ToString())
+          .Concat(targetVariable.ToEnumerable()),
+        Reduce(dataset, rows));
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassification.cs	(revision 9070)
@@ -0,0 +1,96 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nearest neighbour classification data analysis algorithm.
+  /// </summary>
+  [Item("Nearest Neighbour Classification", "Nearest neighbour classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NearestNeighbourClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string KParameterName = "K";
+    private const string NearestNeighbourClassificationModelResultName = "Nearest neighbour classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int K {
+      get { return KParameter.Value.Value; }
+      set {
+        if (value <= 0) throw new ArgumentException("K must be larger than zero.", "K");
+        else KParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NearestNeighbourClassification(bool deserializing) : base(deserializing) { }
+    private NearestNeighbourClassification(NearestNeighbourClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourClassification()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourClassification(this, cloner);
+    }
+
+    #region nearest neighbour
+    protected override void Run() {
+      var solution = CreateNearestNeighbourClassificationSolution(Problem.ProblemData, K);
+      Results.Add(new Result(NearestNeighbourClassificationModelResultName, "The nearest neighbour classification solution.", solution));
+    }
+
+    public static IClassificationSolution CreateNearestNeighbourClassificationSolution(IClassificationProblemData problemData, int k) {
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NearestNeighbourClassificationSolution(problemDataClone, Train(problemDataClone, k));
+    }
+
+    public static INearestNeighbourModel Train(IClassificationProblemData problemData, int k) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables,
+        problemData.ClassValues.ToArray());
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NearestNeighbourClassificationSolution", "Represents a nearest neighbour solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NearestNeighbourClassificationSolution : ClassificationSolution, INearestNeighbourClassificationSolution {
+
+    public new INearestNeighbourModel Model {
+      get { return (INearestNeighbourModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NearestNeighbourClassificationSolution(bool deserializing) : base(deserializing) { }
+    private NearestNeighbourClassificationSolution(NearestNeighbourClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourClassificationSolution(IClassificationProblemData problemData, INearestNeighbourModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourModel.cs	(revision 9070)
@@ -0,0 +1,343 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour model for regression and classification
+  /// </summary>
+  [StorableClass]
+  [Item("NearestNeighbourModel", "Represents a nearest neighbour model for regression and classification.")]
+  public sealed class NearestNeighbourModel : NamedItem, INearestNeighbourModel {
+
+    private alglib.nearestneighbor.kdtree kdTree;
+    public alglib.nearestneighbor.kdtree KDTree {
+      get { return kdTree; }
+      set {
+        if (value != kdTree) {
+          if (value == null) throw new ArgumentNullException();
+          kdTree = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [Storable]
+    private int k;
+
+    [StorableConstructor]
+    private NearestNeighbourModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        kdTree = new alglib.nearestneighbor.kdtree();
+    }
+    private NearestNeighbourModel(NearestNeighbourModel original, Cloner cloner)
+      : base(original, cloner) {
+      kdTree = new alglib.nearestneighbor.kdtree();
+      kdTree.approxf = original.kdTree.approxf;
+      kdTree.boxmax = (double[])original.kdTree.boxmax.Clone();
+      kdTree.boxmin = (double[])original.kdTree.boxmin.Clone();
+      kdTree.buf = (double[])original.kdTree.buf.Clone();
+      kdTree.curboxmax = (double[])original.kdTree.curboxmax.Clone();
+      kdTree.curboxmin = (double[])original.kdTree.curboxmin.Clone();
+      kdTree.curdist = original.kdTree.curdist;
+      kdTree.debugcounter = original.kdTree.debugcounter;
+      kdTree.idx = (int[])original.kdTree.idx.Clone();
+      kdTree.kcur = original.kdTree.kcur;
+      kdTree.kneeded = original.kdTree.kneeded;
+      kdTree.n = original.kdTree.n;
+      kdTree.nodes = (int[])original.kdTree.nodes.Clone();
+      kdTree.normtype = original.kdTree.normtype;
+      kdTree.nx = original.kdTree.nx;
+      kdTree.ny = original.kdTree.ny;
+      kdTree.r = (double[])original.kdTree.r.Clone();
+      kdTree.rneeded = original.kdTree.rneeded;
+      kdTree.selfmatch = original.kdTree.selfmatch;
+      kdTree.splits = (double[])original.kdTree.splits.Clone();
+      kdTree.tags = (int[])original.kdTree.tags.Clone();
+      kdTree.x = (double[])original.kdTree.x.Clone();
+      kdTree.xy = (double[,])original.kdTree.xy.Clone();
+
+      k = original.k;
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NearestNeighbourModel(Dataset dataset, IEnumerable<int> rows, int k, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null) {
+      Name = ItemName;
+      Description = ItemDescription;
+      this.k = k;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+
+      var inputMatrix = AlglibUtil.PrepareInputMatrix(dataset,
+                                   allowedInputVariables.Concat(new string[] { targetVariable }),
+                                   rows);
+
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException(
+          "Nearest neighbour classification does not support NaN or infinity values in the input dataset.");
+
+      this.kdTree = new alglib.nearestneighbor.kdtree();
+
+      var nRows = inputMatrix.GetLength(0);
+      var nFeatures = inputMatrix.GetLength(1) - 1;
+
+      if (classValues != null) {
+        this.classValues = (double[])classValues.Clone();
+        int nClasses = classValues.Length;
+        // map original class values to values [0..nClasses-1]
+        var classIndices = new Dictionary<double, double>();
+        for (int i = 0; i < nClasses; i++)
+          classIndices[classValues[i]] = i;
+
+        for (int row = 0; row < nRows; row++) {
+          inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+        }
+      }
+      alglib.nearestneighbor.kdtreebuild(inputMatrix, nRows, inputMatrix.GetLength(1) - 1, 1, 2, kdTree);
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+      double[] dists = new double[k];
+      double[,] neighbours = new double[k, columns + 1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        int actNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, false);
+        alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
+        alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
+
+        double distanceWeightedValue = 0.0;
+        double distsSum = 0.0;
+        for (int i = 0; i < actNeighbours; i++) {
+          distanceWeightedValue += neighbours[i, columns] / dists[i];
+          distsSum += 1.0 / dists[i];
+        }
+        yield return distanceWeightedValue / distsSum;
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      if (classValues == null) throw new InvalidOperationException("No class values are defined.");
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      int[] y = new int[classValues.Length];
+      double[] dists = new double[k];
+      double[,] neighbours = new double[k, columns + 1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        int actNeighbours = alglib.nearestneighbor.kdtreequeryknn(kdTree, x, k, false);
+        alglib.nearestneighbor.kdtreequeryresultsdistances(kdTree, ref dists);
+        alglib.nearestneighbor.kdtreequeryresultsxy(kdTree, ref neighbours);
+
+        Array.Clear(y, 0, y.Length);
+        for (int i = 0; i < actNeighbours; i++) {
+          int classValue = (int)Math.Round(neighbours[i, columns]);
+          y[classValue]++;
+        }
+
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public INearestNeighbourRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NearestNeighbourRegressionSolution(new RegressionProblemData(problemData), this);
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    public INearestNeighbourClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NearestNeighbourClassificationSolution(new ClassificationProblemData(problemData), this);
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    public double KDTreeApproxF {
+      get { return kdTree.approxf; }
+      set { kdTree.approxf = value; }
+    }
+    [Storable]
+    public double[] KDTreeBoxMax {
+      get { return kdTree.boxmax; }
+      set { kdTree.boxmax = value; }
+    }
+    [Storable]
+    public double[] KDTreeBoxMin {
+      get { return kdTree.boxmin; }
+      set { kdTree.boxmin = value; }
+    }
+    [Storable]
+    public double[] KDTreeBuf {
+      get { return kdTree.buf; }
+      set { kdTree.buf = value; }
+    }
+    [Storable]
+    public double[] KDTreeCurBoxMax {
+      get { return kdTree.curboxmax; }
+      set { kdTree.curboxmax = value; }
+    }
+    [Storable]
+    public double[] KDTreeCurBoxMin {
+      get { return kdTree.curboxmin; }
+      set { kdTree.curboxmin = value; }
+    }
+    [Storable]
+    public double KDTreeCurDist {
+      get { return kdTree.curdist; }
+      set { kdTree.curdist = value; }
+    }
+    [Storable]
+    public int KDTreeDebugCounter {
+      get { return kdTree.debugcounter; }
+      set { kdTree.debugcounter = value; }
+    }
+    [Storable]
+    public int[] KDTreeIdx {
+      get { return kdTree.idx; }
+      set { kdTree.idx = value; }
+    }
+    [Storable]
+    public int KDTreeKCur {
+      get { return kdTree.kcur; }
+      set { kdTree.kcur = value; }
+    }
+    [Storable]
+    public int KDTreeKNeeded {
+      get { return kdTree.kneeded; }
+      set { kdTree.kneeded = value; }
+    }
+    [Storable]
+    public int KDTreeN {
+      get { return kdTree.n; }
+      set { kdTree.n = value; }
+    }
+    [Storable]
+    public int[] KDTreeNodes {
+      get { return kdTree.nodes; }
+      set { kdTree.nodes = value; }
+    }
+    [Storable]
+    public int KDTreeNormType {
+      get { return kdTree.normtype; }
+      set { kdTree.normtype = value; }
+    }
+    [Storable]
+    public int KDTreeNX {
+      get { return kdTree.nx; }
+      set { kdTree.nx = value; }
+    }
+    [Storable]
+    public int KDTreeNY {
+      get { return kdTree.ny; }
+      set { kdTree.ny = value; }
+    }
+    [Storable]
+    public double[] KDTreeR {
+      get { return kdTree.r; }
+      set { kdTree.r = value; }
+    }
+    [Storable]
+    public double KDTreeRNeeded {
+      get { return kdTree.rneeded; }
+      set { kdTree.rneeded = value; }
+    }
+    [Storable]
+    public bool KDTreeSelfMatch {
+      get { return kdTree.selfmatch; }
+      set { kdTree.selfmatch = value; }
+    }
+    [Storable]
+    public double[] KDTreeSplits {
+      get { return kdTree.splits; }
+      set { kdTree.splits = value; }
+    }
+    [Storable]
+    public int[] KDTreeTags {
+      get { return kdTree.tags; }
+      set { kdTree.tags = value; }
+    }
+    [Storable]
+    public double[] KDTreeX {
+      get { return kdTree.x; }
+      set { kdTree.x = value; }
+    }
+    [Storable]
+    public double[,] KDTreeXY {
+      get { return kdTree.xy; }
+      set { kdTree.xy = value; }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegression.cs	(revision 9070)
@@ -0,0 +1,94 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Nearest neighbour regression data analysis algorithm.
+  /// </summary>
+  [Item("Nearest Neighbour Regression", "Nearest neighbour regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NearestNeighbourRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string KParameterName = "K";
+    private const string NearestNeighbourRegressionModelResultName = "Nearest neighbour regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> KParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int K {
+      get { return KParameter.Value.Value; }
+      set {
+        if (value <= 0) throw new ArgumentException("K must be larger than zero.", "K");
+        else KParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private NearestNeighbourRegression(bool deserializing) : base(deserializing) { }
+    private NearestNeighbourRegression(NearestNeighbourRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourRegression()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The number of nearest neighbours to consider for regression.", new IntValue(3)));
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourRegression(this, cloner);
+    }
+
+    #region nearest neighbour
+    protected override void Run() {
+      var solution = CreateNearestNeighbourRegressionSolution(Problem.ProblemData, K);
+      Results.Add(new Result(NearestNeighbourRegressionModelResultName, "The nearest neighbour regression solution.", solution));
+    }
+
+    public static IRegressionSolution CreateNearestNeighbourRegressionSolution(IRegressionProblemData problemData, int k) {
+      var clonedProblemData = (IRegressionProblemData)problemData.Clone();
+      return new NearestNeighbourRegressionSolution(clonedProblemData, Train(problemData, k));
+    }
+
+    public static INearestNeighbourModel Train(IRegressionProblemData problemData, int k) {
+      return new NearestNeighbourModel(problemData.Dataset,
+        problemData.TrainingIndices,
+        k,
+        problemData.TargetVariable,
+        problemData.AllowedInputVariables);
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NearestNeighbour/NearestNeighbourRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a nearest neighbour solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NearestNeighbourRegressionSolution", "Represents a nearest neighbour solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NearestNeighbourRegressionSolution : RegressionSolution, INearestNeighbourRegressionSolution {
+
+    public new INearestNeighbourModel Model {
+      get { return (INearestNeighbourModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NearestNeighbourRegressionSolution(bool deserializing) : base(deserializing) { }
+    private NearestNeighbourRegressionSolution(NearestNeighbourRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NearestNeighbourRegressionSolution(IRegressionProblemData problemData, INearestNeighbourModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NearestNeighbourRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 9070)
@@ -0,0 +1,229 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network classification data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkRegressionModelResultName = "Neural network classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkClassification(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkClassification(NeuralNetworkClassification original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public NeuralNetworkClassification()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+        (IntValue)new IntValue(0).AsReadOnly(), 
+        (IntValue)new IntValue(1).AsReadOnly(), 
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(1)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. This value is not used if the number of hidden layers is zero.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      RestartsParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+
+      RegisterEventHandlers();
+
+      Problem = new ClassificationProblem();
+    }
+
+    private void RegisterEventHandlers() {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameter.ValueChanged += HiddenLayersParameterValueChanged;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkClassification(this, cloner);
+    }
+    private void HiddenLayersParameterValueChanged(object source, EventArgs e) {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameterValueValueChanged(this, EventArgs.Empty);
+    }
+
+    private void HiddenLayersParameterValueValueChanged(object source, EventArgs e) {
+      if (HiddenLayers == 0) {
+        NodesInFirstHiddenLayerParameter.Hidden = true;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else if (HiddenLayers == 1) {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = false;
+      }
+    }
+
+    #region neural network
+    protected override void Run() {
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkClassificationSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
+      Results.Add(new Result(NeuralNetworkRegressionModelResultName, "The neural network regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateNeuralNetworkClassificationSolution(IClassificationProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network classification does not support NaN or infinity values in the input dataset.");
+
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+
+      alglib.multilayerperceptron multiLayerPerceptron = null;
+      if (nLayers == 0) {
+        alglib.mlpcreatec0(allowedInputVariables.Count(), nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 1) {
+        alglib.mlpcreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 2) {
+        alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, out multiLayerPerceptron);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+
+      int info;
+      // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
+      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 2) throw new ArgumentException("Error in calculation of neural network classification solution");
+
+      rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
+      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+      relClassError = alglib.mlpclserror(multiLayerPerceptron, inputMatrix, nRows) / (double)nRows;
+
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NeuralNetworkClassificationSolution(problemDataClone, new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,54 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkClassificationSolution", "Represents a neural network solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkClassificationSolution : ClassificationSolution, INeuralNetworkClassificationSolution {
+
+    public new INeuralNetworkModel Model {
+      get { return (INeuralNetworkModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkClassificationSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkClassificationSolution(NeuralNetworkClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkClassificationSolution(IClassificationProblemData problemData, INeuralNetworkModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkClassificationSolution(this, cloner);
+    }
+
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 9070)
@@ -0,0 +1,213 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble classification data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Classification", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleClassificationModelResultName = "Neural network ensemble classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassification(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleClassification(NeuralNetworkEnsembleClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassification()
+      : base() {
+        var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+        (IntValue)new IntValue(0).AsReadOnly(), 
+        (IntValue)new IntValue(1).AsReadOnly(), 
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      HiddenLayersParameter.Hidden = true;
+      NodesInFirstHiddenLayerParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+      RestartsParameter.Hidden = true;
+
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassification(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run() {
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkEnsembleClassificationSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
+      Results.Add(new Result(NeuralNetworkEnsembleClassificationModelResultName, "The neural network ensemble classification solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateNeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network ensemble classification does not support NaN or infinity values in the input dataset.");
+
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetDoubleValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndices[inputMatrix[row, nFeatures]];
+      }
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreatec0(allowedInputVariables.Count(), nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble regression solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+      relClassError = alglib.mlperelclserror(mlpEnsemble, inputMatrix, nRows);
+      var problemDataClone = (IClassificationProblemData)problemData.Clone();
+      return new NeuralNetworkEnsembleClassificationSolution(problemDataClone, new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables, problemDataClone.ClassValues.ToArray()));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleClassificationSolution", "Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleClassificationSolution : ClassificationSolution, INeuralNetworkEnsembleClassificationSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassificationSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleClassificationSolution(NeuralNetworkEnsembleClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 9070)
@@ -0,0 +1,235 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensembel model for regression and classification
+  /// </summary>
+  [StorableClass]
+  [Item("NeuralNetworkEnsembleModel", "Represents a neural network ensemble for regression and classification.")]
+  public sealed class NeuralNetworkEnsembleModel : NamedItem, INeuralNetworkEnsembleModel {
+
+    private alglib.mlpensemble mlpEnsemble;
+    public alglib.mlpensemble MultiLayerPerceptronEnsemble {
+      get { return mlpEnsemble; }
+      set {
+        if (value != mlpEnsemble) {
+          if (value == null) throw new ArgumentNullException();
+          mlpEnsemble = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private NeuralNetworkEnsembleModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        mlpEnsemble = new alglib.mlpensemble();
+    }
+    private NeuralNetworkEnsembleModel(NeuralNetworkEnsembleModel original, Cloner cloner)
+      : base(original, cloner) {
+      mlpEnsemble = new alglib.mlpensemble();
+      string serializedEnsemble;
+      alglib.mlpeserialize(original.mlpEnsemble, out serializedEnsemble);
+      alglib.mlpeunserialize(serializedEnsemble, out this.mlpEnsemble);
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NeuralNetworkEnsembleModel(alglib.mlpensemble mlpEnsemble, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.mlpEnsemble = mlpEnsemble;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        yield return y[0];
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public INeuralNetworkEnsembleRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkEnsembleRegressionSolution(new RegressionEnsembleProblemData(problemData), this);
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    public INeuralNetworkEnsembleClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkEnsembleClassificationSolution(new ClassificationEnsembleProblemData(problemData), this);
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private string MultiLayerPerceptronEnsembleNetwork {
+      get {
+        string serializedNetwork;
+        alglib.mlpeserialize(this.mlpEnsemble, out serializedNetwork);
+        return serializedNetwork;
+      }
+      set {
+        alglib.mlpeunserialize(value, out this.mlpEnsemble);
+      }
+    }
+
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnMeans {
+      get { return mlpEnsemble.innerobj.columnmeans; }
+      set {
+        mlpEnsemble.innerobj.columnmeans = value;
+        mlpEnsemble.innerobj.network.columnmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnSigmas {
+      get { return mlpEnsemble.innerobj.columnsigmas; }
+      set {
+        mlpEnsemble.innerobj.columnsigmas = value;
+        mlpEnsemble.innerobj.network.columnsigmas = value;
+      }
+    }
+    [Storable(AllowOneWay = true)]
+    private double[] MultiLayerPerceptronEnsembleDfdnet {
+      set {
+        mlpEnsemble.innerobj.network.dfdnet = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleSize {
+      get { return mlpEnsemble.innerobj.ensemblesize; }
+      set {
+        mlpEnsemble.innerobj.ensemblesize = value;
+        mlpEnsemble.innerobj.ensemblesize = value;
+      }
+    }
+    [Storable(AllowOneWay = true)]
+    private double[] MultiLayerPerceptronEnsembleNeurons {
+      set { mlpEnsemble.innerobj.network.neurons = value; }
+    }
+    [Storable(AllowOneWay = true)]
+    private double[] MultiLayerPerceptronEnsembleSerializedMlp {
+      set {
+        mlpEnsemble.innerobj.network.dfdnet = value;
+      }
+    }
+    [Storable(AllowOneWay = true)]
+    private int[] MultiLayerPerceptronStuctinfo {
+      set {
+        mlpEnsemble.innerobj.network.structinfo = value;
+      }
+    }
+
+    [Storable]
+    private double[] MultiLayerPerceptronWeights {
+      get {
+        return mlpEnsemble.innerobj.weights;
+      }
+      set {
+        mlpEnsemble.innerobj.weights = value;
+        mlpEnsemble.innerobj.network.weights = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronY {
+      get {
+        return mlpEnsemble.innerobj.y;
+      }
+      set {
+        mlpEnsemble.innerobj.y = value;
+        mlpEnsemble.innerobj.network.y = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 9070)
@@ -0,0 +1,199 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble regression data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Regression", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleRegressionModelResultName = "Neural network ensemble regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegression(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleRegression(NeuralNetworkEnsembleRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegression()
+      : base() {
+        var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+        (IntValue)new IntValue(0).AsReadOnly(), 
+        (IntValue)new IntValue(1).AsReadOnly(), 
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      HiddenLayersParameter.Hidden = true;
+      NodesInFirstHiddenLayerParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+      RestartsParameter.Hidden = true;
+
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegression(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run() {
+      double rmsError, avgRelError;
+      var solution = CreateNeuralNetworkEnsembleRegressionSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      Results.Add(new Result(NeuralNetworkEnsembleRegressionModelResultName, "The neural network ensemble regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble regression solution on the training set.", new PercentValue(avgRelError)));
+    }
+
+    public static IRegressionSolution CreateNeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network ensemble regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreate0(allowedInputVariables.Count(), 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreate1(allowedInputVariables.Count(), nHiddenNodes1, 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+      int nRows = inputMatrix.GetLength(0);
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble regression solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+
+      return new NeuralNetworkEnsembleRegressionSolution((IRegressionProblemData)problemData.Clone(), new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,54 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleRegressionSolution", "Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleRegressionSolution : RegressionSolution, INeuralNetworkEnsembleRegressionSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegressionSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleRegressionSolution(NeuralNetworkEnsembleRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+      : base(nnModel, problemData) {
+      RecalculateResults();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkModel.cs	(revision 9070)
@@ -0,0 +1,263 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network model for regression and classification
+  /// </summary>
+  [StorableClass]
+  [Item("NeuralNetworkModel", "Represents a neural network for regression and classification.")]
+  public sealed class NeuralNetworkModel : NamedItem, INeuralNetworkModel {
+
+    private alglib.multilayerperceptron multiLayerPerceptron;
+    public alglib.multilayerperceptron MultiLayerPerceptron {
+      get { return multiLayerPerceptron; }
+      set {
+        if (value != multiLayerPerceptron) {
+          if (value == null) throw new ArgumentNullException();
+          multiLayerPerceptron = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private NeuralNetworkModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        multiLayerPerceptron = new alglib.multilayerperceptron();
+    }
+    private NeuralNetworkModel(NeuralNetworkModel original, Cloner cloner)
+      : base(original, cloner) {
+      multiLayerPerceptron = new alglib.multilayerperceptron();
+      multiLayerPerceptron.innerobj.chunks = (double[,])original.multiLayerPerceptron.innerobj.chunks.Clone();
+      multiLayerPerceptron.innerobj.columnmeans = (double[])original.multiLayerPerceptron.innerobj.columnmeans.Clone();
+      multiLayerPerceptron.innerobj.columnsigmas = (double[])original.multiLayerPerceptron.innerobj.columnsigmas.Clone();
+      multiLayerPerceptron.innerobj.derror = (double[])original.multiLayerPerceptron.innerobj.derror.Clone();
+      multiLayerPerceptron.innerobj.dfdnet = (double[])original.multiLayerPerceptron.innerobj.dfdnet.Clone();
+      multiLayerPerceptron.innerobj.neurons = (double[])original.multiLayerPerceptron.innerobj.neurons.Clone();
+      multiLayerPerceptron.innerobj.nwbuf = (double[])original.multiLayerPerceptron.innerobj.nwbuf.Clone();
+      multiLayerPerceptron.innerobj.structinfo = (int[])original.multiLayerPerceptron.innerobj.structinfo.Clone();
+      multiLayerPerceptron.innerobj.weights = (double[])original.multiLayerPerceptron.innerobj.weights.Clone();
+      multiLayerPerceptron.innerobj.x = (double[])original.multiLayerPerceptron.innerobj.x.Clone();
+      multiLayerPerceptron.innerobj.y = (double[])original.multiLayerPerceptron.innerobj.y.Clone();
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NeuralNetworkModel(alglib.multilayerperceptron multiLayerPerceptron, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.multiLayerPerceptron = multiLayerPerceptron;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpprocess(multiLayerPerceptron, x, ref y);
+        yield return y[0];
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpprocess(multiLayerPerceptron, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public INeuralNetworkRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new NeuralNetworkRegressionSolution(new RegressionProblemData(problemData), this);
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    public INeuralNetworkClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new NeuralNetworkClassificationSolution(new ClassificationProblemData(problemData), this);
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private double[,] MultiLayerPerceptronChunks {
+      get {
+        return multiLayerPerceptron.innerobj.chunks;
+      }
+      set {
+        multiLayerPerceptron.innerobj.chunks = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronColumnMeans {
+      get {
+        return multiLayerPerceptron.innerobj.columnmeans;
+      }
+      set {
+        multiLayerPerceptron.innerobj.columnmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronColumnSigmas {
+      get {
+        return multiLayerPerceptron.innerobj.columnsigmas;
+      }
+      set {
+        multiLayerPerceptron.innerobj.columnsigmas = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronDError {
+      get {
+        return multiLayerPerceptron.innerobj.derror;
+      }
+      set {
+        multiLayerPerceptron.innerobj.derror = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronDfdnet {
+      get {
+        return multiLayerPerceptron.innerobj.dfdnet;
+      }
+      set {
+        multiLayerPerceptron.innerobj.dfdnet = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronNeurons {
+      get {
+        return multiLayerPerceptron.innerobj.neurons;
+      }
+      set {
+        multiLayerPerceptron.innerobj.neurons = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronNwbuf {
+      get {
+        return multiLayerPerceptron.innerobj.nwbuf;
+      }
+      set {
+        multiLayerPerceptron.innerobj.nwbuf = value;
+      }
+    }
+    [Storable]
+    private int[] MultiLayerPerceptronStuctinfo {
+      get {
+        return multiLayerPerceptron.innerobj.structinfo;
+      }
+      set {
+        multiLayerPerceptron.innerobj.structinfo = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronWeights {
+      get {
+        return multiLayerPerceptron.innerobj.weights;
+      }
+      set {
+        multiLayerPerceptron.innerobj.weights = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronX {
+      get {
+        return multiLayerPerceptron.innerobj.x;
+      }
+      set {
+        multiLayerPerceptron.innerobj.x = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronY {
+      get {
+        return multiLayerPerceptron.innerobj.y;
+      }
+      set {
+        multiLayerPerceptron.innerobj.y = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 9070)
@@ -0,0 +1,213 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network regression data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkRegressionModelResultName = "Neural network regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public IConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (IConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkRegression(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkRegression(NeuralNetworkRegression original, Cloner cloner)
+      : base(original, cloner) {
+      RegisterEventHandlers();
+    }
+    public NeuralNetworkRegression()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { 
+        (IntValue)new IntValue(0).AsReadOnly(), 
+        (IntValue)new IntValue(1).AsReadOnly(), 
+        (IntValue)new IntValue(2).AsReadOnly() });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(1)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. This value is not used if the number of hidden layers is zero.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      RestartsParameter.Hidden = true;
+      NodesInSecondHiddenLayerParameter.Hidden = true;
+
+      RegisterEventHandlers();
+
+      Problem = new RegressionProblem();
+    }
+
+    private void RegisterEventHandlers() {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameter.ValueChanged += HiddenLayersParameterValueChanged;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      RegisterEventHandlers();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkRegression(this, cloner);
+    }
+
+    private void HiddenLayersParameterValueChanged(object source, EventArgs e) {
+      HiddenLayersParameter.Value.ValueChanged += HiddenLayersParameterValueValueChanged;
+      HiddenLayersParameterValueValueChanged(this, EventArgs.Empty);
+    }
+
+    private void HiddenLayersParameterValueValueChanged(object source, EventArgs e) {
+      if (HiddenLayers == 0) {
+        NodesInFirstHiddenLayerParameter.Hidden = true;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else if (HiddenLayers == 1) {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = true;
+      } else {
+        NodesInFirstHiddenLayerParameter.Hidden = false;
+        NodesInSecondHiddenLayerParameter.Hidden = false;
+      }
+    }
+
+
+    #region neural network
+    protected override void Run() {
+      double rmsError, avgRelError;
+      var solution = CreateNeuralNetworkRegressionSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      Results.Add(new Result(NeuralNetworkRegressionModelResultName, "The neural network regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network regression solution on the training set.", new PercentValue(avgRelError)));
+    }
+
+    public static IRegressionSolution CreateNeuralNetworkRegressionSolution(IRegressionProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.multilayerperceptron multiLayerPerceptron = null;
+      if (nLayers == 0) {
+        alglib.mlpcreate0(allowedInputVariables.Count(), 1, out multiLayerPerceptron);
+      } else if (nLayers == 1) {
+        alglib.mlpcreate1(allowedInputVariables.Count(), nHiddenNodes1, 1, out multiLayerPerceptron);
+      } else if (nLayers == 2) {
+        alglib.mlpcreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, out multiLayerPerceptron);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+      int nRows = inputMatrix.GetLength(0);
+
+      int info;
+      // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
+      alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 2) throw new ArgumentException("Error in calculation of neural network regression solution");
+
+      rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
+      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+
+      return new NeuralNetworkRegressionSolution((IRegressionProblemData)problemData.Clone(), new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkRegressionSolution", "Represents a neural network solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkRegressionSolution : RegressionSolution, INeuralNetworkRegressionSolution {
+
+    public new INeuralNetworkModel Model {
+      get { return (INeuralNetworkModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkRegressionSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkRegressionSolution(NeuralNetworkRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkRegressionSolution(IRegressionProblemData problemData, INeuralNetworkModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Plugin class for HeuristicLab.Algorithms.DataAnalysis plugin.
+  /// </summary>
+  [Plugin("HeuristicLab.Algorithms.DataAnalysis", "Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)", "3.4.3.$WCREV$")]
+  [PluginFile("HeuristicLab.Algorithms.DataAnalysis-3.4.dll", PluginFileType.Assembly)]
+  [PluginDependency("HeuristicLab.ALGLIB", "3.6.0")]
+  [PluginDependency("HeuristicLab.Algorithms.GradientDescent", "3.3")]
+  [PluginDependency("HeuristicLab.Analysis", "3.3")]
+  [PluginDependency("HeuristicLab.Collections", "3.3")]
+  [PluginDependency("HeuristicLab.Common", "3.3")]
+  [PluginDependency("HeuristicLab.Common.Resources", "3.3")]
+  [PluginDependency("HeuristicLab.Core", "3.3")]
+  [PluginDependency("HeuristicLab.Data", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.RealVectorEncoding", "3.3")]
+  [PluginDependency("HeuristicLab.Encodings.SymbolicExpressionTreeEncoding", "3.4")]
+  [PluginDependency("HeuristicLab.Operators", "3.3")]
+  [PluginDependency("HeuristicLab.Optimization", "3.3")]
+  [PluginDependency("HeuristicLab.Parameters", "3.3")]
+  [PluginDependency("HeuristicLab.Persistence", "3.3")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Classification", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.Regression", "3.4")]
+  [PluginDependency("HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis", "3.4")]
+  [PluginDependency("HeuristicLab.LibSVM", "3.12")]
+  [PluginDependency("HeuristicLab.Random", "3.3")]
+  public class HeuristicLabAlgorithmsDataAnalysisPlugin : PluginBase {
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/Properties/AssemblyInfo.cs.frame	(revision 9070)
@@ -0,0 +1,55 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.DataAnalysis")]
+[assembly: AssemblyDescription("Provides wrappers for data analysis algorithms implemented in external libraries (linear regression, linear discriminant analysis, k-means clustering, support vector classification and regression)")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("HeuristicLab")]
+[assembly: AssemblyCopyright("(c) 2002-2012 HEAL")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("3E422463-0CE0-400D-9544-5C11D5818769")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Revision and Build Numbers 
+// by using the '*' as shown below:
+[assembly: AssemblyVersion("3.4.0.0")]
+[assembly: AssemblyFileVersion("3.4.3.$WCREV$")]
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassification.cs	(revision 9070)
@@ -0,0 +1,177 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Random forest classification data analysis algorithm.
+  /// </summary>
+  [Item("Random Forest Classification", "Random forest classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class RandomForestClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string RandomForestClassificationModelResultName = "Random forest classification solution";
+    private const string NumberOfTreesParameterName = "Number of trees";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> NumberOfTreesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NumberOfTreesParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int NumberOfTrees {
+      get { return NumberOfTreesParameter.Value.Value; }
+      set { NumberOfTreesParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    private RandomForestClassification(bool deserializing) : base(deserializing) { }
+    private RandomForestClassification(RandomForestClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public RandomForestClassification()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(NumberOfTreesParameterName, "The number of trees in the forest. Should be between 50 and 100", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "The ratio of the training set that will be used in the construction of individual trees (0<r<=1). Should be adjusted depending on the noise level in the dataset in the range from 0.66 (low noise) to 0.05 (high noise). This parameter should be adjusted to achieve good generalization error.", new DoubleValue(0.3)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Problem = new ClassificationProblem();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(MParameterName))
+        Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      if (!Parameters.ContainsKey(SeedParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
+        Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestClassification(this, cloner);
+    }
+
+    #region random forest
+    protected override void Run() {
+      double rmsError, relClassificationError, outOfBagRmsError, outOfBagRelClassificationError;
+      if (SetSeedRandomly) Seed = new System.Random().Next();
+
+      var solution = CreateRandomForestClassificationSolution(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out relClassificationError, out outOfBagRmsError, out outOfBagRelClassificationError);
+      Results.Add(new Result(RandomForestClassificationModelResultName, "The random forest classification solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Relative classification error", "Relative classification error of the random forest regression solution on the training set.", new PercentValue(relClassificationError)));
+      Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
+      Results.Add(new Result("Relative classification error (out-of-bag)", "The out-of-bag relative classification error  of the random forest regression solution.", new PercentValue(outOfBagRelClassificationError)));
+    }
+
+    public static IClassificationSolution CreateRandomForestClassificationSolution(IClassificationProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double relClassificationError, out double outOfBagRmsError, out double outOfBagRelClassificationError) {
+      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
+      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
+
+      alglib.math.rndobject = new System.Random(seed);
+
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Random forest classification does not support NaN or infinity values in the input dataset.");
+
+      int info = 0;
+      alglib.decisionforest dForest = new alglib.decisionforest();
+      alglib.dfreport rep = new alglib.dfreport(); ;
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
+      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
+
+
+      double[] classValues = problemData.ClassValues.ToArray();
+      int nClasses = problemData.Classes;
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndices = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndices[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nColumns - 1] = classIndices[inputMatrix[row, nColumns - 1]];
+      }
+      // execute random forest algorithm      
+      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, nClasses, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      if (info != 1) throw new ArgumentException("Error in calculation of random forest classification solution");
+
+      rmsError = rep.rmserror;
+      outOfBagRmsError = rep.oobrmserror;
+      relClassificationError = rep.relclserror;
+      outOfBagRelClassificationError = rep.oobrelclserror;
+      return new RandomForestClassificationSolution((IClassificationProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables, classValues));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("RandomForestClassificationSolution", "Represents a random forest solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class RandomForestClassificationSolution : ClassificationSolution, IRandomForestClassificationSolution {
+
+    public new IRandomForestModel Model {
+      get { return (IRandomForestModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private RandomForestClassificationSolution(bool deserializing) : base(deserializing) { }
+    private RandomForestClassificationSolution(RandomForestClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public RandomForestClassificationSolution(IClassificationProblemData problemData, IRandomForestModel randomForestModel)
+      : base(randomForestModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs	(revision 9070)
@@ -0,0 +1,203 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest model for regression and classification
+  /// </summary>
+  [StorableClass]
+  [Item("RandomForestModel", "Represents a random forest for regression and classification.")]
+  public sealed class RandomForestModel : NamedItem, IRandomForestModel {
+
+    private alglib.decisionforest randomForest;
+    public alglib.decisionforest RandomForest {
+      get { return randomForest; }
+      set {
+        if (value != randomForest) {
+          if (value == null) throw new ArgumentNullException();
+          randomForest = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private RandomForestModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        randomForest = new alglib.decisionforest();
+    }
+    private RandomForestModel(RandomForestModel original, Cloner cloner)
+      : base(original, cloner) {
+      randomForest = new alglib.decisionforest();
+      randomForest.innerobj.bufsize = original.randomForest.innerobj.bufsize;
+      randomForest.innerobj.nclasses = original.randomForest.innerobj.nclasses;
+      randomForest.innerobj.ntrees = original.randomForest.innerobj.ntrees;
+      randomForest.innerobj.nvars = original.randomForest.innerobj.nvars;
+      randomForest.innerobj.trees = (double[])original.randomForest.innerobj.trees.Clone();
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public RandomForestModel(alglib.decisionforest randomForest, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.randomForest = randomForest;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.dfprocess(randomForest, x, ref y);
+        yield return y[0];
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[randomForest.innerobj.nclasses];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.dfprocess(randomForest, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    public IRandomForestRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new RandomForestRegressionSolution(new RegressionProblemData(problemData), this);
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    public IRandomForestClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new RandomForestClassificationSolution(new ClassificationProblemData(problemData), this);
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private int RandomForestBufSize {
+      get {
+        return randomForest.innerobj.bufsize;
+      }
+      set {
+        randomForest.innerobj.bufsize = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNClasses {
+      get {
+        return randomForest.innerobj.nclasses;
+      }
+      set {
+        randomForest.innerobj.nclasses = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNTrees {
+      get {
+        return randomForest.innerobj.ntrees;
+      }
+      set {
+        randomForest.innerobj.ntrees = value;
+      }
+    }
+    [Storable]
+    private int RandomForestNVars {
+      get {
+        return randomForest.innerobj.nvars;
+      }
+      set {
+        randomForest.innerobj.nvars = value;
+      }
+    }
+    [Storable]
+    private double[] RandomForestTrees {
+      get {
+        return randomForest.innerobj.trees;
+      }
+      set {
+        randomForest.innerobj.trees = value;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegression.cs	(revision 9070)
@@ -0,0 +1,165 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Random forest regression data analysis algorithm.
+  /// </summary>
+  [Item("Random Forest Regression", "Random forest regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class RandomForestRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string RandomForestRegressionModelResultName = "Random forest regression solution";
+    private const string NumberOfTreesParameterName = "Number of trees";
+    private const string RParameterName = "R";
+    private const string MParameterName = "M";
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> NumberOfTreesParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NumberOfTreesParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> RParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> MParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[MParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+    #endregion
+    #region properties
+    public int NumberOfTrees {
+      get { return NumberOfTreesParameter.Value.Value; }
+      set { NumberOfTreesParameter.Value.Value = value; }
+    }
+    public double R {
+      get { return RParameter.Value.Value; }
+      set { RParameter.Value.Value = value; }
+    }
+    public double M {
+      get { return MParameter.Value.Value; }
+      set { MParameter.Value.Value = value; }
+    }
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private RandomForestRegression(bool deserializing) : base(deserializing) { }
+    private RandomForestRegression(RandomForestRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+
+    public RandomForestRegression()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(NumberOfTreesParameterName, "The number of trees in the forest. Should be between 50 and 100", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RParameterName, "The ratio of the training set that will be used in the construction of individual trees (0<r<=1). Should be adjusted depending on the noise level in the dataset in the range from 0.66 (low noise) to 0.05 (high noise). This parameter should be adjusted to achieve good generalization error.", new DoubleValue(0.3)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+      Problem = new RegressionProblem();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(MParameterName))
+        Parameters.Add(new FixedValueParameter<DoubleValue>(MParameterName, "The ratio of features that will be used in the construction of individual trees (0<m<=1)", new DoubleValue(0.5)));
+      if (!Parameters.ContainsKey(SeedParameterName))
+        Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+      if (!Parameters.ContainsKey((SetSeedRandomlyParameterName)))
+        Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestRegression(this, cloner);
+    }
+
+    #region random forest
+    protected override void Run() {
+      double rmsError, avgRelError, outOfBagRmsError, outOfBagAvgRelError;
+      if (SetSeedRandomly) Seed = new System.Random().Next();
+
+      var solution = CreateRandomForestRegressionSolution(Problem.ProblemData, NumberOfTrees, R, M, Seed, out rmsError, out avgRelError, out outOfBagRmsError, out outOfBagAvgRelError);
+      Results.Add(new Result(RandomForestRegressionModelResultName, "The random forest regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the random forest regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the random forest regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Root mean square error (out-of-bag)", "The out-of-bag root of the mean of squared errors of the random forest regression solution.", new DoubleValue(outOfBagRmsError)));
+      Results.Add(new Result("Average relative error (out-of-bag)", "The out-of-bag average of relative errors of the random forest regression solution.", new PercentValue(outOfBagAvgRelError)));
+    }
+
+    public static IRegressionSolution CreateRandomForestRegressionSolution(IRegressionProblemData problemData, int nTrees, double r, double m, int seed,
+      out double rmsError, out double avgRelError, out double outOfBagRmsError, out double outOfBagAvgRelError) {
+      if (r <= 0 || r > 1) throw new ArgumentException("The R parameter in the random forest regression must be between 0 and 1.");
+      if (m <= 0 || m > 1) throw new ArgumentException("The M parameter in the random forest regression must be between 0 and 1.");
+
+      alglib.math.rndobject = new System.Random(seed);
+
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Random forest regression does not support NaN or infinity values in the input dataset.");
+
+      int info = 0;
+      alglib.decisionforest dForest = new alglib.decisionforest();
+      alglib.dfreport rep = new alglib.dfreport(); ;
+      int nRows = inputMatrix.GetLength(0);
+      int nColumns = inputMatrix.GetLength(1);
+      int sampleSize = Math.Max((int)Math.Round(r * nRows), 1);
+      int nFeatures = Math.Max((int)Math.Round(m * (nColumns - 1)), 1);
+
+      alglib.dforest.dfbuildinternal(inputMatrix, nRows, nColumns - 1, 1, nTrees, sampleSize, nFeatures, alglib.dforest.dfusestrongsplits + alglib.dforest.dfuseevs, ref info, dForest.innerobj, rep.innerobj);
+      if (info != 1) throw new ArgumentException("Error in calculation of random forest regression solution");
+
+      rmsError = rep.rmserror;
+      avgRelError = rep.avgrelerror;
+      outOfBagAvgRelError = rep.oobavgrelerror;
+      outOfBagRmsError = rep.oobrmserror;
+
+      return new RandomForestRegressionSolution((IRegressionProblemData)problemData.Clone(), new RandomForestModel(dForest, targetVariable, allowedInputVariables));
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a random forest solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("RandomForestRegressionSolution", "Represents a random forest solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class RandomForestRegressionSolution : RegressionSolution, IRandomForestRegressionSolution {
+
+    public new IRandomForestModel Model {
+      get { return (IRandomForestModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private RandomForestRegressionSolution(bool deserializing) : base(deserializing) { }
+    private RandomForestRegressionSolution(RandomForestRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public RandomForestRegressionSolution(IRegressionProblemData problemData, IRandomForestModel randomForestModel)
+      : base(randomForestModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new RandomForestRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs	(revision 9070)
@@ -0,0 +1,210 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Support vector machine classification data analysis algorithm.
+  /// </summary>
+  [Item("Support Vector Classification", "Support vector machine classification data analysis algorithm (wrapper for libSVM).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class SupportVectorClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string SvmTypeParameterName = "SvmType";
+    private const string KernelTypeParameterName = "KernelType";
+    private const string CostParameterName = "Cost";
+    private const string NuParameterName = "Nu";
+    private const string GammaParameterName = "Gamma";
+    private const string DegreeParameterName = "Degree";
+
+    #region parameter properties
+    public IConstrainedValueParameter<StringValue> SvmTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[SvmTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> KernelTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[KernelTypeParameterName]; }
+    }
+    public IValueParameter<DoubleValue> NuParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IValueParameter<DoubleValue> CostParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[CostParameterName]; }
+    }
+    public IValueParameter<DoubleValue> GammaParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[GammaParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    #endregion
+    #region properties
+    public StringValue SvmType {
+      get { return SvmTypeParameter.Value; }
+      set { SvmTypeParameter.Value = value; }
+    }
+    public StringValue KernelType {
+      get { return KernelTypeParameter.Value; }
+      set { KernelTypeParameter.Value = value; }
+    }
+    public DoubleValue Nu {
+      get { return NuParameter.Value; }
+    }
+    public DoubleValue Cost {
+      get { return CostParameter.Value; }
+    }
+    public DoubleValue Gamma {
+      get { return GammaParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private SupportVectorClassification(bool deserializing) : base(deserializing) { }
+    private SupportVectorClassification(SupportVectorClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorClassification()
+      : base() {
+      Problem = new ClassificationProblem();
+
+      List<StringValue> svrTypes = (from type in new List<string> { "NU_SVC", "C_SVC" }
+                                    select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> svrTypeSet = new ItemSet<StringValue>(svrTypes);
+      List<StringValue> kernelTypes = (from type in new List<string> { "LINEAR", "POLY", "SIGMOID", "RBF" }
+                                       select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> kernelTypeSet = new ItemSet<StringValue>(kernelTypes);
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(SvmTypeParameterName, "The type of SVM to use.", svrTypeSet, svrTypes[0]));
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(KernelTypeParameterName, "The kernel type to use for the SVM.", kernelTypeSet, kernelTypes[3]));
+      Parameters.Add(new ValueParameter<DoubleValue>(NuParameterName, "The value of the nu parameter nu-SVC.", new DoubleValue(0.5)));
+      Parameters.Add(new ValueParameter<DoubleValue>(CostParameterName, "The value of the C (cost) parameter of C-SVC.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+      if (!Parameters.ContainsKey(DegreeParameterName))
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorClassification(this, cloner);
+    }
+
+    #region support vector classification
+    protected override void Run() {
+      IClassificationProblemData problemData = Problem.ProblemData;
+      IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
+      double trainingAccuracy, testAccuracy;
+      int nSv;
+      var solution = CreateSupportVectorClassificationSolution(problemData, selectedInputVariables,
+        SvmType.Value, KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Degree.Value,
+        out trainingAccuracy, out testAccuracy, out nSv);
+
+      Results.Add(new Result("Support vector classification solution", "The support vector classification solution.", solution));
+      Results.Add(new Result("Training accuracy", "The accuracy of the SVR solution on the training partition.", new DoubleValue(trainingAccuracy)));
+      Results.Add(new Result("Test accuracy", "The accuracy of the SVR solution on the test partition.", new DoubleValue(testAccuracy)));
+      Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
+    }
+
+    public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, int degree,
+      out double trainingAccuracy, out double testAccuracy, out int nSv) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+
+      //extract SVM parameters from scope and set them
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = GetSvmType(svmType);
+      parameter.kernel_type = GetKernelType(kernelType);
+      parameter.C = cost;
+      parameter.nu = nu;
+      parameter.gamma = gamma;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = degree;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+
+
+      var weightLabels = new List<int>();
+      var weights = new List<double>();
+      foreach (double c in problemData.ClassValues) {
+        double wSum = 0.0;
+        foreach (double otherClass in problemData.ClassValues) {
+          if (!c.IsAlmost(otherClass)) {
+            wSum += problemData.GetClassificationPenalty(c, otherClass);
+          }
+        }
+        weightLabels.Add((int)c);
+        weights.Add(wSum);
+      }
+      parameter.weight_label = weightLabels.ToArray();
+      parameter.weight = weights.ToArray();
+
+
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
+      var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables, problemData.ClassValues);
+      var solution = new SupportVectorClassificationSolution(model, (IClassificationProblemData)problemData.Clone());
+
+      nSv = svmModel.SV.Length;
+      trainingAccuracy = solution.TrainingAccuracy;
+      testAccuracy = solution.TestAccuracy;
+
+      return solution;
+    }
+
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVC") return svm_parameter.NU_SVC;
+      if (svmType == "C_SVC") return svm_parameter.C_SVC;
+      throw new ArgumentException("Unknown SVM type");
+    }
+
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassificationSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("SupportVectorClassificationSolution", "Represents a support vector solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class SupportVectorClassificationSolution : ClassificationSolution, ISupportVectorMachineSolution {
+
+    public new ISupportVectorMachineModel Model {
+      get { return (ISupportVectorMachineModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private SupportVectorClassificationSolution(bool deserializing) : base(deserializing) { }
+    private SupportVectorClassificationSolution(SupportVectorClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorClassificationSolution(SupportVectorMachineModel model, IClassificationProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorClassificationSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineModel.cs	(revision 9070)
@@ -0,0 +1,215 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector machine model.
+  /// </summary>
+  [StorableClass]
+  [Item("SupportVectorMachineModel", "Represents a support vector machine model.")]
+  public sealed class SupportVectorMachineModel : NamedItem, ISupportVectorMachineModel {
+
+    private svm_model model;
+    /// <summary>
+    /// Gets or sets the SVM model.
+    /// </summary>
+    public svm_model Model {
+      get { return model; }
+      set {
+        if (value != model) {
+          if (value == null) throw new ArgumentNullException();
+          model = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    /// <summary>
+    /// Gets or sets the range transformation for the model.
+    /// </summary>
+    private RangeTransform rangeTransform;
+    public RangeTransform RangeTransform {
+      get { return rangeTransform; }
+      set {
+        if (value != rangeTransform) {
+          if (value == null) throw new ArgumentNullException();
+          rangeTransform = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    public Dataset SupportVectors {
+      get {
+        var data = new double[Model.sv_coef.Length, allowedInputVariables.Count()];
+        for (int i = 0; i < Model.sv_coef.Length; i++) {
+          var sv = Model.SV[i];
+          for (int j = 0; j < sv.Length; j++) {
+            data[i, j] = sv[j].value;
+          }
+        }
+        return new Dataset(allowedInputVariables, data);
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues; // only for SVM classification models
+
+    [StorableConstructor]
+    private SupportVectorMachineModel(bool deserializing) : base(deserializing) { }
+    private SupportVectorMachineModel(SupportVectorMachineModel original, Cloner cloner)
+      : base(original, cloner) {
+      // only using a shallow copy here! (gkronber)
+      this.model = original.model;
+      this.rangeTransform = original.rangeTransform;
+      this.targetVariable = original.targetVariable;
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables, IEnumerable<double> classValues)
+      : this(model, rangeTransform, targetVariable, allowedInputVariables) {
+      this.classValues = classValues.ToArray();
+    }
+    public SupportVectorMachineModel(svm_model model, RangeTransform rangeTransform, string targetVariable, IEnumerable<string> allowedInputVariables)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.model = model;
+      this.rangeTransform = rangeTransform;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorMachineModel(this, cloner);
+    }
+
+    #region IRegressionModel Members
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      return GetEstimatedValuesHelper(dataset, rows);
+    }
+    public SupportVectorRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      return new SupportVectorRegressionSolution(this, new RegressionProblemData(problemData));
+    }
+    IRegressionSolution IRegressionModel.CreateRegressionSolution(IRegressionProblemData problemData) {
+      return CreateRegressionSolution(problemData);
+    }
+    #endregion
+
+    #region IClassificationModel Members
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      if (classValues == null) throw new NotSupportedException();
+      // return the original class value instead of the predicted value of the model
+      // svm classification only works for integer classes
+      foreach (var estimated in GetEstimatedValuesHelper(dataset, rows)) {
+        // find closest class
+        double bestDist = double.MaxValue;
+        double bestClass = -1;
+        for (int i = 0; i < classValues.Length; i++) {
+          double d = Math.Abs(estimated - classValues[i]);
+          if (d < bestDist) {
+            bestDist = d;
+            bestClass = classValues[i];
+            if (d.IsAlmost(0.0)) break; // exact match no need to look further
+          }
+        }
+        yield return bestClass;
+      }
+    }
+
+    public SupportVectorClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+      return new SupportVectorClassificationSolution(this, new ClassificationProblemData(problemData));
+    }
+    IClassificationSolution IClassificationModel.CreateClassificationSolution(IClassificationProblemData problemData) {
+      return CreateClassificationSolution(problemData);
+    }
+    #endregion
+    private IEnumerable<double> GetEstimatedValuesHelper(Dataset dataset, IEnumerable<int> rows) {
+      // calculate predictions for the currently requested rows
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+
+      for (int i = 0; i < problem.l; i++) {
+        yield return svm.svm_predict(Model, scaledProblem.x[i]);
+      }
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private string ModelAsString {
+      get {
+        using (MemoryStream stream = new MemoryStream()) {
+          svm.svm_save_model(new StreamWriter(stream), Model);
+          stream.Seek(0, System.IO.SeekOrigin.Begin);
+          StreamReader reader = new StreamReader(stream);
+          return reader.ReadToEnd();
+        }
+      }
+      set {
+        using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
+          model = svm.svm_load_model(new StreamReader(stream));
+        }
+      }
+    }
+    [Storable]
+    private string RangeTransformAsString {
+      get {
+        using (MemoryStream stream = new MemoryStream()) {
+          RangeTransform.Write(stream, RangeTransform);
+          stream.Seek(0, System.IO.SeekOrigin.Begin);
+          StreamReader reader = new StreamReader(stream);
+          return reader.ReadToEnd();
+        }
+      }
+      set {
+        using (MemoryStream stream = new MemoryStream(Encoding.ASCII.GetBytes(value))) {
+          RangeTransform = RangeTransform.Read(stream);
+        }
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs	(revision 9070)
@@ -0,0 +1,63 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public class SupportVectorMachineUtil {
+    /// <summary>
+    /// Transforms <paramref name="problemData"/> into a data structure as needed by libSVM.
+    /// </summary>
+    /// <param name="problemData">The problem data to transform</param>
+    /// <param name="rowIndices">The rows of the dataset that should be contained in the resulting SVM-problem</param>
+    /// <returns>A problem data type that can be used to train a support vector machine.</returns>
+    public static svm_problem CreateSvmProblem(Dataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
+      double[] targetVector =
+        dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
+
+      svm_node[][] nodes = new svm_node[targetVector.Length][];
+      List<svm_node> tempRow;
+      int maxNodeIndex = 0;
+      int svmProblemRowIndex = 0;
+      List<string> inputVariablesList = inputVariables.ToList();
+      foreach (int row in rowIndices) {
+        tempRow = new List<svm_node>();
+        int colIndex = 1; // make sure the smallest node index for SVM = 1
+        foreach (var inputVariable in inputVariablesList) {
+          double value = dataset.GetDoubleValue(inputVariable, row);
+          // SVM also works with missing values
+          // => don't add NaN values in the dataset to the sparse SVM matrix representation
+          if (!double.IsNaN(value)) {
+            tempRow.Add(new svm_node() { index = colIndex, value = value }); // nodes must be sorted in ascending ordered by column index
+            if (colIndex > maxNodeIndex) maxNodeIndex = colIndex;
+          }
+          colIndex++;
+        }
+        nodes[svmProblemRowIndex++] = tempRow.ToArray();
+      }
+
+      return new svm_problem() { l = targetVector.Length, y = targetVector, x = nodes };
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegression.cs	(revision 9070)
@@ -0,0 +1,202 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using LibSVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Support vector machine regression data analysis algorithm.
+  /// </summary>
+  [Item("Support Vector Regression", "Support vector machine regression data analysis algorithm (wrapper for libSVM).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class SupportVectorRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string SvmTypeParameterName = "SvmType";
+    private const string KernelTypeParameterName = "KernelType";
+    private const string CostParameterName = "Cost";
+    private const string NuParameterName = "Nu";
+    private const string GammaParameterName = "Gamma";
+    private const string EpsilonParameterName = "Epsilon";
+    private const string DegreeParameterName = "Degree";
+
+    #region parameter properties
+    public IConstrainedValueParameter<StringValue> SvmTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[SvmTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> KernelTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[KernelTypeParameterName]; }
+    }
+    public IValueParameter<DoubleValue> NuParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[NuParameterName]; }
+    }
+    public IValueParameter<DoubleValue> CostParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[CostParameterName]; }
+    }
+    public IValueParameter<DoubleValue> GammaParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[GammaParameterName]; }
+    }
+    public IValueParameter<DoubleValue> EpsilonParameter {
+      get { return (IValueParameter<DoubleValue>)Parameters[EpsilonParameterName]; }
+    }
+    public IValueParameter<IntValue> DegreeParameter {
+      get { return (IValueParameter<IntValue>)Parameters[DegreeParameterName]; }
+    }
+    #endregion
+    #region properties
+    public StringValue SvmType {
+      get { return SvmTypeParameter.Value; }
+      set { SvmTypeParameter.Value = value; }
+    }
+    public StringValue KernelType {
+      get { return KernelTypeParameter.Value; }
+      set { KernelTypeParameter.Value = value; }
+    }
+    public DoubleValue Nu {
+      get { return NuParameter.Value; }
+    }
+    public DoubleValue Cost {
+      get { return CostParameter.Value; }
+    }
+    public DoubleValue Gamma {
+      get { return GammaParameter.Value; }
+    }
+    public DoubleValue Epsilon {
+      get { return EpsilonParameter.Value; }
+    }
+    public IntValue Degree {
+      get { return DegreeParameter.Value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private SupportVectorRegression(bool deserializing) : base(deserializing) { }
+    private SupportVectorRegression(SupportVectorRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorRegression()
+      : base() {
+      Problem = new RegressionProblem();
+
+      List<StringValue> svrTypes = (from type in new List<string> { "NU_SVR", "EPSILON_SVR" }
+                                    select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> svrTypeSet = new ItemSet<StringValue>(svrTypes);
+      List<StringValue> kernelTypes = (from type in new List<string> { "LINEAR", "POLY", "SIGMOID", "RBF" }
+                                       select new StringValue(type).AsReadOnly())
+                                   .ToList();
+      ItemSet<StringValue> kernelTypeSet = new ItemSet<StringValue>(kernelTypes);
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(SvmTypeParameterName, "The type of SVM to use.", svrTypeSet, svrTypes[0]));
+      Parameters.Add(new ConstrainedValueParameter<StringValue>(KernelTypeParameterName, "The kernel type to use for the SVM.", kernelTypeSet, kernelTypes[3]));
+      Parameters.Add(new ValueParameter<DoubleValue>(NuParameterName, "The value of the nu parameter of the nu-SVR.", new DoubleValue(0.5)));
+      Parameters.Add(new ValueParameter<DoubleValue>(CostParameterName, "The value of the C (cost) parameter of epsilon-SVR and nu-SVR.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(GammaParameterName, "The value of the gamma parameter in the kernel function.", new DoubleValue(1.0)));
+      Parameters.Add(new ValueParameter<DoubleValue>(EpsilonParameterName, "The value of the epsilon parameter for epsilon-SVR.", new DoubleValue(0.1)));
+      Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      #region backwards compatibility (change with 3.4)
+      if (!Parameters.ContainsKey(DegreeParameterName))
+        Parameters.Add(new ValueParameter<IntValue>(DegreeParameterName, "The degree parameter for the polynomial kernel function.", new IntValue(3)));
+      #endregion
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorRegression(this, cloner);
+    }
+
+    #region support vector regression
+    protected override void Run() {
+      IRegressionProblemData problemData = Problem.ProblemData;
+      IEnumerable<string> selectedInputVariables = problemData.AllowedInputVariables;
+      double trainR2, testR2;
+      int nSv;
+      var solution = CreateSupportVectorRegressionSolution(problemData, selectedInputVariables, SvmType.Value,
+        KernelType.Value, Cost.Value, Nu.Value, Gamma.Value, Epsilon.Value, Degree.Value,
+        out trainR2, out testR2, out nSv);
+
+      Results.Add(new Result("Support vector regression solution", "The support vector regression solution.", solution));
+      Results.Add(new Result("Training R²", "The Pearson's R² of the SVR solution on the training partition.", new DoubleValue(trainR2)));
+      Results.Add(new Result("Test R²", "The Pearson's R² of the SVR solution on the test partition.", new DoubleValue(testR2)));
+      Results.Add(new Result("Number of support vectors", "The number of support vectors of the SVR solution.", new IntValue(nSv)));
+    }
+
+    public static SupportVectorRegressionSolution CreateSupportVectorRegressionSolution(IRegressionProblemData problemData, IEnumerable<string> allowedInputVariables,
+      string svmType, string kernelType, double cost, double nu, double gamma, double epsilon, int degree,
+      out double trainingR2, out double testR2, out int nSv) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+
+      //extract SVM parameters from scope and set them
+      svm_parameter parameter = new svm_parameter();
+      parameter.svm_type = GetSvmType(svmType);
+      parameter.kernel_type = GetKernelType(kernelType);
+      parameter.C = cost;
+      parameter.nu = nu;
+      parameter.gamma = gamma;
+      parameter.p = epsilon;
+      parameter.cache_size = 500;
+      parameter.probability = 0;
+      parameter.eps = 0.001;
+      parameter.degree = degree;
+      parameter.shrinking = 1;
+      parameter.coef0 = 0;
+
+
+
+      svm_problem problem = SupportVectorMachineUtil.CreateSvmProblem(dataset, targetVariable, allowedInputVariables, rows);
+      RangeTransform rangeTransform = RangeTransform.Compute(problem);
+      svm_problem scaledProblem = rangeTransform.Scale(problem);
+      var svmModel = svm.svm_train(scaledProblem, parameter);
+      nSv = svmModel.SV.Length;
+      var model = new SupportVectorMachineModel(svmModel, rangeTransform, targetVariable, allowedInputVariables);
+      var solution = new SupportVectorRegressionSolution(model, (IRegressionProblemData)problemData.Clone());
+      trainingR2 = solution.TrainingRSquared;
+      testR2 = solution.TestRSquared;
+      return solution;
+    }
+
+    private static int GetSvmType(string svmType) {
+      if (svmType == "NU_SVR") return svm_parameter.NU_SVR;
+      if (svmType == "EPSILON_SVR") return svm_parameter.EPSILON_SVR;
+      throw new ArgumentException("Unknown SVM type");
+    }
+
+    private static int GetKernelType(string kernelType) {
+      if (kernelType == "LINEAR") return svm_parameter.LINEAR;
+      if (kernelType == "POLY") return svm_parameter.POLY;
+      if (kernelType == "SIGMOID") return svm_parameter.SIGMOID;
+      if (kernelType == "RBF") return svm_parameter.RBF;
+      throw new ArgumentException("Unknown kernel type");
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorRegressionSolution.cs	(revision 9070)
@@ -0,0 +1,53 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a support vector solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("SupportVectorRegressionSolution", "Represents a support vector solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class SupportVectorRegressionSolution : RegressionSolution, ISupportVectorMachineSolution {
+
+    public new ISupportVectorMachineModel Model {
+      get { return (ISupportVectorMachineModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private SupportVectorRegressionSolution(bool deserializing) : base(deserializing) { }
+    private SupportVectorRegressionSolution(SupportVectorRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public SupportVectorRegressionSolution(SupportVectorMachineModel model, IRegressionProblemData problemData)
+      : base(model, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new SupportVectorRegressionSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs	(revision 9070)
@@ -0,0 +1,141 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.TimeSeriesPrognosis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.TimeSeries {
+  [Item("Autoregressive Modeling", "Timeseries modeling algorithm that creates AR-N models.")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public class AutoregressiveModeling : FixedDataAnalysisAlgorithm<ITimeSeriesPrognosisProblem> {
+    private const string TimeOffesetParameterName = "Maximum Time Offset";
+
+    public IFixedValueParameter<IntValue> TimeOffsetParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[TimeOffesetParameterName]; }
+    }
+
+
+    public int TimeOffset {
+      get { return TimeOffsetParameter.Value.Value; }
+      set { TimeOffsetParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    protected AutoregressiveModeling(bool deserializing) : base(deserializing) { }
+    protected AutoregressiveModeling(AutoregressiveModeling original, Cloner cloner) : base(original, cloner) { }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new AutoregressiveModeling(this, cloner);
+    }
+
+    public AutoregressiveModeling()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(TimeOffesetParameterName, "The maximum time offset for the model ranging from 1 to infinity.", new IntValue(1)));
+      Problem = new TimeSeriesPrognosisProblem();
+    }
+
+    protected override void Run() {
+      double rmsError, cvRmsError;
+      var solution = CreateAutoRegressiveSolution(Problem.ProblemData, TimeOffset, out rmsError, out cvRmsError);
+      Results.Add(new Result("Autoregressive solution", "The autoregressive time series prognosis solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the autoregressive time series prognosis solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Estimated root mean square error (cross-validation)", "The estimated root of the mean of squared errors of the autoregressive time series prognosis solution via cross validation.", new DoubleValue(cvRmsError)));
+    }
+
+    /// <summary>
+    /// Calculates an AR(p) model. For further information see http://en.wikipedia.org/wiki/Autoregressive_model
+    /// </summary>
+    /// <param name="problemData">The problem data which should be used for training</param>
+    /// <param name="timeOffset">The parameter p of the AR(p) specifying the maximum time offset [1,infinity] </param>
+    /// <returns>The times series autoregressive solution </returns>
+    public static ITimeSeriesPrognosisSolution CreateAutoRegressiveSolution(ITimeSeriesPrognosisProblemData problemData, int timeOffset) {
+      double rmsError, cvRmsError;
+      return CreateAutoRegressiveSolution(problemData, timeOffset, out rmsError, out cvRmsError);
+    }
+
+    private static ITimeSeriesPrognosisSolution CreateAutoRegressiveSolution(ITimeSeriesPrognosisProblemData problemData, int timeOffset, out double rmsError, out double cvRmsError) {
+      string targetVariable = problemData.TargetVariable;
+
+      double[,] inputMatrix = new double[problemData.TrainingPartition.Size, timeOffset + 1];
+      var targetValues = problemData.Dataset.GetDoubleValues(targetVariable).ToList();
+      for (int i = 0, row = problemData.TrainingPartition.Start; i < problemData.TrainingPartition.Size; i++, row++) {
+        for (int col = 0; col < timeOffset; col++) {
+          inputMatrix[i, col] = targetValues[row - col - 1];
+        }
+      }
+      // set target values in last column
+      for (int i = 0; i < inputMatrix.GetLength(0); i++)
+        inputMatrix[i, timeOffset] = targetValues[i + problemData.TrainingPartition.Start];
+
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Linear regression does not support NaN or infinity values in the input dataset.");
+
+
+      alglib.linearmodel lm = new alglib.linearmodel();
+      alglib.lrreport ar = new alglib.lrreport();
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
+
+      int retVal = 1;
+      alglib.lrbuild(inputMatrix, nRows, nFeatures, out retVal, out lm, out ar);
+      if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+      rmsError = ar.rmserror;
+      cvRmsError = ar.cvrmserror;
+
+      alglib.lrunpack(lm, out coefficients, out nFeatures);
+
+
+      ISymbolicExpressionTree tree = new SymbolicExpressionTree(new ProgramRootSymbol().CreateTreeNode());
+      ISymbolicExpressionTreeNode startNode = new StartSymbol().CreateTreeNode();
+      tree.Root.AddSubtree(startNode);
+      ISymbolicExpressionTreeNode addition = new Addition().CreateTreeNode();
+      startNode.AddSubtree(addition);
+
+      for (int i = 0; i < timeOffset; i++) {
+        LaggedVariableTreeNode node = (LaggedVariableTreeNode)new LaggedVariable().CreateTreeNode();
+        node.VariableName = targetVariable;
+        node.Weight = coefficients[i];
+        node.Lag = (i + 1) * -1;
+        addition.AddSubtree(node);
+      }
+
+      ConstantTreeNode cNode = (ConstantTreeNode)new Constant().CreateTreeNode();
+      cNode.Value = coefficients[coefficients.Length - 1];
+      addition.AddSubtree(cNode);
+
+      var interpreter = new SymbolicTimeSeriesPrognosisExpressionTreeInterpreter(problemData.TargetVariable);
+      var model = new SymbolicTimeSeriesPrognosisModel(tree, interpreter);
+      var solution = model.CreateTimeSeriesPrognosisSolution((ITimeSeriesPrognosisProblemData)problemData.Clone());
+      return solution;
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClustering.cs	(revision 9070)
@@ -0,0 +1,103 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// k-Means clustering algorithm data analysis algorithm.
+  /// </summary>
+  [Item("k-Means", "The k-Means clustering algorithm (wrapper for ALGLIB).")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class KMeansClustering : FixedDataAnalysisAlgorithm<IClusteringProblem> {
+    private const string KParameterName = "k";
+    private const string RestartsParameterName = "Restarts";
+    private const string KMeansSolutionResultName = "k-Means clustering solution";
+    #region parameter properties
+    public IValueParameter<IntValue> KParameter {
+      get { return (IValueParameter<IntValue>)Parameters[KParameterName]; }
+    }
+    public IValueParameter<IntValue> RestartsParameter {
+      get { return (IValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+    #region properties
+    public IntValue K {
+      get { return KParameter.Value; }
+    }
+    public IntValue Restarts {
+      get { return RestartsParameter.Value; }
+    }
+    #endregion
+    [StorableConstructor]
+    private KMeansClustering(bool deserializing) : base(deserializing) { }
+    private KMeansClustering(KMeansClustering original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public KMeansClustering()
+      : base() {
+      Parameters.Add(new ValueParameter<IntValue>(KParameterName, "The number of clusters.", new IntValue(3)));
+      Parameters.Add(new ValueParameter<IntValue>(RestartsParameterName, "The number of restarts.", new IntValue(0)));
+      Problem = new ClusteringProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClustering(this, cloner);
+    }
+
+    #region k-Means clustering
+    protected override void Run() {
+      var solution = CreateKMeansSolution(Problem.ProblemData, K.Value, Restarts.Value);
+      Results.Add(new Result(KMeansSolutionResultName, "The k-Means clustering solution.", solution));
+    }
+
+    public static KMeansClusteringSolution CreateKMeansSolution(IClusteringProblemData problemData, int k, int restarts) {
+      Dataset dataset = problemData.Dataset;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndices;
+      int info;
+      double[,] centers;
+      int[] xyc;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("k-Means clustering does not support NaN or infinity values in the input dataset.");
+
+      alglib.kmeansgenerate(inputMatrix, inputMatrix.GetLength(0), inputMatrix.GetLength(1), k, restarts + 1, out info, out centers, out xyc);
+      if (info != 1) throw new ArgumentException("Error in calculation of k-Means clustering solution");
+
+      KMeansClusteringSolution solution = new KMeansClusteringSolution(new KMeansClusteringModel(centers, allowedInputVariables), (IClusteringProblemData)problemData.Clone());
+      return solution;
+    }
+    #endregion
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringModel.cs	(revision 9070)
@@ -0,0 +1,87 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a k-Means clustering model.
+  /// </summary>
+  [StorableClass]
+  [Item("KMeansClusteringModel", "Represents a k-Means clustering model.")]
+  public sealed class KMeansClusteringModel : NamedItem, IClusteringModel {
+    public static new Image StaticItemImage {
+      get { return HeuristicLab.Common.Resources.VSImageLibrary.Function; }
+    }
+
+    [Storable]
+    private string[] allowedInputVariables;
+    public IEnumerable<string> AllowedInputVariables {
+      get { return allowedInputVariables; }
+    }
+    [Storable]
+    private List<double[]> centers;
+    public IEnumerable<double[]> Centers {
+      get {
+        return centers.Select(x => (double[])x.Clone());
+      }
+    }
+    [StorableConstructor]
+    private KMeansClusteringModel(bool deserializing) : base(deserializing) { }
+    private KMeansClusteringModel(KMeansClusteringModel original, Cloner cloner)
+      : base(original, cloner) {
+      this.allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      this.centers = new List<double[]>(original.Centers);
+    }
+    public KMeansClusteringModel(double[,] centers, IEnumerable<string> allowedInputVariables)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      // disect center matrix into list of double[]
+      // centers are given as double matrix where number of rows = dimensions and number of columns = clusters
+      // each column is a cluster center
+      this.centers = new List<double[]>();
+      for (int i = 0; i < centers.GetLength(1); i++) {
+        double[] c = new double[centers.GetLength(0)];
+        for (int j = 0; j < c.Length; j++) {
+          c[j] = centers[j, i];
+        }
+        this.centers.Add(c);
+      }
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClusteringModel(this, cloner);
+    }
+
+
+    public IEnumerable<int> GetClusterValues(Dataset dataset, IEnumerable<int> rows) {
+      return KMeansClusteringUtil.FindClosestCenters(centers, dataset, allowedInputVariables, rows);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringSolution.cs	(revision 9070)
@@ -0,0 +1,64 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Optimization;
+using HeuristicLab.Data;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a k-Means clustering solution for a clustering problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("k-Means clustering solution", "Represents a k-Means solution for a clustering problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class KMeansClusteringSolution : ClusteringSolution {
+    private const string TrainingIntraClusterSumOfSquaresResultName = "Intra-cluster sum of squares (training)";
+    private const string TestIntraClusterSumOfSquaresResultName = "Intra-cluster sum of squares (test)";
+    public new KMeansClusteringModel Model {
+      get { return (KMeansClusteringModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private KMeansClusteringSolution(bool deserializing) : base(deserializing) { }
+    private KMeansClusteringSolution(KMeansClusteringSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public KMeansClusteringSolution(KMeansClusteringModel model, IClusteringProblemData problemData)
+      : base(model, problemData) {
+      double trainingIntraClusterSumOfSquares = KMeansClusteringUtil.CalculateIntraClusterSumOfSquares(model, problemData.Dataset, problemData.TrainingIndices);
+      double testIntraClusterSumOfSquares = KMeansClusteringUtil.CalculateIntraClusterSumOfSquares(model, problemData.Dataset, problemData.TestIndices);
+      this.Add(new Result(TrainingIntraClusterSumOfSquaresResultName, "The sum of squared distances of points of the training partition to the cluster center (is minimized by k-Means).", new DoubleValue(trainingIntraClusterSumOfSquares)));
+      this.Add(new Result(TestIntraClusterSumOfSquaresResultName, "The sum of squared distances of points of the test partition to the cluster center (is minimized by k-Means).", new DoubleValue(testIntraClusterSumOfSquares)));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new KMeansClusteringSolution(this, cloner);
+    }
+  }
+}
Index: /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs
===================================================================
--- /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 9070)
+++ /branches/ClassificationModelComparison/HeuristicLab.Algorithms.DataAnalysis/3.4/kMeans/KMeansClusteringUtil.cs	(revision 9070)
@@ -0,0 +1,111 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public static class KMeansClusteringUtil {
+    public static IEnumerable<int> FindClosestCenters(IEnumerable<double[]> centers, Dataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<int> rows) {
+      int nRows = rows.Count();
+      int nCols = allowedInputVariables.Count();
+      int[] closestCenter = new int[nRows];
+      double[] bestCenterDistance = Enumerable.Repeat(double.MaxValue, nRows).ToArray();
+      int centerIndex = 1;
+
+      foreach (double[] center in centers) {
+        if (nCols != center.Length) throw new ArgumentException();
+        int rowIndex = 0;
+        foreach (var row in rows) {
+          // calc euclidian distance of point to center
+          double centerDistance = 0;
+          int col = 0;
+          foreach (var inputVariable in allowedInputVariables) {
+            double d = center[col++] - dataset.GetDoubleValue(inputVariable, row);
+            d = d * d; // square;
+            centerDistance += d;
+            if (centerDistance > bestCenterDistance[rowIndex]) break;
+          }
+          if (centerDistance < bestCenterDistance[rowIndex]) {
+            bestCenterDistance[rowIndex] = centerDistance;
+            closestCenter[rowIndex] = centerIndex;
+          }
+          rowIndex++;
+        }
+        centerIndex++;
+      }
+      return closestCenter;
+    }
+
+    public static double CalculateIntraClusterSumOfSquares(KMeansClusteringModel model, Dataset dataset, IEnumerable<int> rows) {
+      List<int> clusterValues = model.GetClusterValues(dataset, rows).ToList();
+      List<string> allowedInputVariables = model.AllowedInputVariables.ToList();
+      int nCols = allowedInputVariables.Count;
+      Dictionary<int, List<double[]>> clusterPoints = new Dictionary<int, List<double[]>>();
+      Dictionary<int, double[]> clusterMeans = new Dictionary<int, double[]>();
+      foreach (var clusterValue in clusterValues.Distinct()) {
+        clusterPoints.Add(clusterValue, new List<double[]>());
+      }
+
+      // collect points of clusters
+      int clusterValueIndex = 0;
+      foreach (var row in rows) {
+        double[] p = new double[allowedInputVariables.Count];
+        for (int i = 0; i < nCols; i++) {
+          p[i] = dataset.GetDoubleValue(allowedInputVariables[i], row);
+        }
+        clusterPoints[clusterValues[clusterValueIndex++]].Add(p);
+      }
+      // calculate cluster means
+      foreach (var pair in clusterPoints) {
+        double[] mean = new double[nCols];
+        foreach (var p in pair.Value) {
+          for (int i = 0; i < nCols; i++) {
+            mean[i] += p[i];
+          }
+        }
+        for (int i = 0; i < nCols; i++) {
+          mean[i] /= pair.Value.Count;
+        }
+        clusterMeans[pair.Key] = mean;
+      }
+      // calculate distances
+      double allCenterDistances = 0;
+      foreach (var pair in clusterMeans) {
+        double[] mean = pair.Value;
+        double centerDistances = 0;
+        foreach (var clusterPoint in clusterPoints[pair.Key]) {
+          double centerDistance = 0;
+          for (int i = 0; i < nCols; i++) {
+            double d = mean[i] - clusterPoint[i];
+            d = d * d;
+            centerDistance += d;
+          }
+          centerDistances += centerDistance;
+        }
+        allCenterDistances += centerDistances;
+      }
+      return allCenterDistances;
+    }
+  }
+}
