Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 9270)
@@ -224,10 +224,16 @@
     <Compile Include="Nca\Initialization\INcaInitializer.cs" />
     <Compile Include="Nca\Initialization\LdaInitializer.cs" />
+    <Compile Include="Nca\Initialization\NcaInitializer.cs" />
     <Compile Include="Nca\Initialization\PcaInitializer.cs" />
     <Compile Include="Nca\Initialization\RandomInitializer.cs" />
     <Compile Include="Nca\Matrix.cs" />
+    <Compile Include="Nca\ModelCreation\INcaModelCreator.cs" />
+    <Compile Include="Nca\ModelCreation\NcaModelCreator.cs" />
     <Compile Include="Nca\NcaAlgorithm.cs" />
     <Compile Include="Nca\NcaClassificationSolution.cs" />
+    <Compile Include="Nca\NcaGradientCalculator.cs" />
     <Compile Include="Nca\NcaModel.cs" />
+    <Compile Include="Nca\SolutionCreation\INcaSolutionCreator.cs" />
+    <Compile Include="Nca\SolutionCreation\NcaSolutionCreator.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourClassification.cs" />
     <Compile Include="NearestNeighbour\NearestNeighbourClassificationSolution.cs" />
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/INcaInitializer.cs	(revision 9270)
@@ -24,5 +24,5 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  public interface INCAInitializer : IItem {
+  public interface INcaInitializer : IOperator {
     /// <summary>
     /// Calculates an initial projection for the NCA to start from.
@@ -30,6 +30,6 @@
     /// <param name="data">The problem data that contains the AllowedInputVariables and TrainingIndices.</param>
     /// <param name="dimensions">The amount of columns in the matrix</param>
-    /// <returns>A flat representation of a matrix that is read row-wise and contains AllowedInputVariables * TrainingIndices numbers.</returns>
-    double[] Initialize(IClassificationProblemData data, int dimensions);
+    /// <returns>The matrix that projects the input variables into a lower dimensional space.</returns>
+    double[,] Initialize(IClassificationProblemData data, Scaling scaling, int dimensions);
   }
 }
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/LdaInitializer.cs	(revision 9270)
@@ -20,5 +20,4 @@
 #endregion
 
-using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Common;
@@ -30,49 +29,40 @@
   [Item("LDA", "Initializes the matrix by performing a linear discriminant analysis.")]
   [StorableClass]
-  public class LDAInitializer : Item, INCAInitializer {
+  public class LdaInitializer : NcaInitializer {
 
     [StorableConstructor]
-    protected LDAInitializer(bool deserializing) : base(deserializing) { }
-    protected LDAInitializer(LDAInitializer original, Cloner cloner) : base(original, cloner) { }
-    public LDAInitializer() : base() { }
+    protected LdaInitializer(bool deserializing) : base(deserializing) { }
+    protected LdaInitializer(LdaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public LdaInitializer() : base() { }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new LDAInitializer(this, cloner);
+      return new LdaInitializer(this, cloner);
     }
 
-    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+    public override double[,] Initialize(IClassificationProblemData data, Scaling scaling, int dimensions) {
       var instances = data.TrainingIndices.Count();
       var attributes = data.AllowedInputVariables.Count();
 
       var ldaDs = new double[instances, attributes + 1];
-      int row, col = 0;
-      foreach (var variable in data.AllowedInputVariables) {
-        row = 0;
-        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
-          ldaDs[row, col] = value;
-          row++;
+      int j = 0;
+      foreach (var a in data.AllowedInputVariables) {
+        int i = 0;
+        var sv = scaling.GetScaledValues(data.Dataset, a, data.TrainingIndices);
+        foreach (var v in sv) {
+          ldaDs[i++, j] = v;
         }
-        col++;
+        j++;
       }
-      row = 0;
-      var uniqueClasses = new Dictionary<double, int>();
-      foreach (var label in data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices)) {
-        if (!uniqueClasses.ContainsKey(label))
-          uniqueClasses[label] = uniqueClasses.Count;
-        ldaDs[row++, attributes] = label;
-      }
-      for (row = 0; row < instances; row++)
-        ldaDs[row, attributes] = uniqueClasses[ldaDs[row, attributes]];
+      j = 0;
+      foreach (var tv in data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices))
+        ldaDs[j++, attributes] = tv;
+
+      var uniqueClasses = data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices).Distinct().Count();
 
       int info;
       double[,] matrix;
-      alglib.fisherldan(ldaDs, instances, attributes, uniqueClasses.Count, out info, out matrix);
+      alglib.fisherldan(ldaDs, instances, attributes, uniqueClasses, out info, out matrix);
 
-      var result = new double[attributes * dimensions];
-      for (int i = 0; i < attributes; i++)
-        for (int j = 0; j < dimensions; j++)
-          result[i * dimensions + j] = matrix[i, j];
-
-      return result;
+      return matrix;
     }
 
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/NcaInitializer.cs	(revision 9270)
@@ -0,0 +1,79 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaInitializer", "Base class for initializers for NCA.")]
+  [StorableClass]
+  public abstract class NcaInitializer : SingleSuccessorOperator, INcaInitializer {
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+    public ILookupParameter<Scaling> ScalingParameter {
+      get { return (ILookupParameter<Scaling>)Parameters["Scaling"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaInitializer(bool deserializing) : base(deserializing) { }
+    protected NcaInitializer(NcaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public NcaInitializer() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The number of dimensions to which the features should be pruned."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The coefficients of the matrix that need to be optimized. Note that the matrix is flattened."));
+      Parameters.Add(new LookupParameter<Scaling>("Scaling", "Each dataset is scaled and the information is stored in a scaling object."));
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+      var scaling = new Scaling(problemData.Dataset, problemData.AllowedInputVariables, problemData.TrainingIndices);
+
+      var dimensions = DimensionsParameter.ActualValue.Value;
+      var matrix = Initialize(problemData, scaling, dimensions);
+      var attributes = matrix.GetLength(0);
+
+      var result = new double[attributes * dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          result[i * dimensions + j] = matrix[i, j];
+
+      ScalingParameter.ActualValue = scaling;
+      NcaMatrixParameter.ActualValue = new RealVector(result);
+      return base.Apply();
+    }
+
+    public abstract double[,] Initialize(IClassificationProblemData data, Scaling scaling, int dimensions);
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/PcaInitializer.cs	(revision 9270)
@@ -29,29 +29,20 @@
   [Item("PCA", "Initializes the matrix by performing a principal components analysis.")]
   [StorableClass]
-  public sealed class PCAInitializer : Item, INCAInitializer {
+  public sealed class PcaInitializer : NcaInitializer {
 
     [StorableConstructor]
-    private PCAInitializer(bool deserializing) : base(deserializing) { }
-    private PCAInitializer(PCAInitializer original, Cloner cloner) : base(original, cloner) { }
-    public PCAInitializer() : base() { }
+    private PcaInitializer(bool deserializing) : base(deserializing) { }
+    private PcaInitializer(PcaInitializer original, Cloner cloner) : base(original, cloner) { }
+    public PcaInitializer() : base() { }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new PCAInitializer(this, cloner);
+      return new PcaInitializer(this, cloner);
     }
 
-    public double[] Initialize(IClassificationProblemData data, int dimensions) {
+    public override double[,] Initialize(IClassificationProblemData data, Scaling scaling, int dimensions) {
       var instances = data.TrainingIndices.Count();
       var attributes = data.AllowedInputVariables.Count();
 
-      var pcaDs = new double[instances, attributes];
-      int col = 0;
-      foreach (var variable in data.AllowedInputVariables) {
-        int row = 0;
-        foreach (var value in data.Dataset.GetDoubleValues(variable, data.TrainingIndices)) {
-          pcaDs[row, col] = value;
-          row++;
-        }
-        col++;
-      }
+      var pcaDs = AlglibUtil.PrepareAndScaleInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices, scaling);
 
       int info;
@@ -60,10 +51,5 @@
       alglib.pcabuildbasis(pcaDs, instances, attributes, out info, out varianceValues, out matrix);
 
-      var result = new double[attributes * dimensions];
-      for (int i = 0; i < attributes; i++)
-        for (int j = 0; j < dimensions; j++)
-          result[i * dimensions + j] = matrix[i, j];
-
-      return result;
+      return matrix;
     }
 
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/Initialization/RandomInitializer.cs	(revision 9270)
@@ -23,38 +23,23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Data;
+using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Random;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [Item("Random", "Initializes the matrix randomly.")]
   [StorableClass]
-  public class RandomInitializer : ParameterizedNamedItem, INCAInitializer {
-    private IValueParameter<IntValue> RandomParameter {
-      get { return (IValueParameter<IntValue>)Parameters["Seed"]; }
-    }
-    private IValueParameter<BoolValue> SetSeedRandomlyParameter {
-      get { return (IValueParameter<BoolValue>)Parameters["SetSeedRandomly"]; }
-    }
-
-    public int Seed {
-      get { return RandomParameter.Value.Value; }
-      set { RandomParameter.Value.Value = value; }
-    }
-
-    public bool SetSeedRandomly {
-      get { return SetSeedRandomlyParameter.Value.Value; }
-      set { SetSeedRandomlyParameter.Value.Value = value; }
+  public sealed class RandomInitializer : NcaInitializer, IStochasticOperator {
+    public ILookupParameter<IRandom> RandomParameter {
+      get { return (ILookupParameter<IRandom>)Parameters["Random"]; }
     }
 
     [StorableConstructor]
-    protected RandomInitializer(bool deserializing) : base(deserializing) { }
-    protected RandomInitializer(RandomInitializer original, Cloner cloner) : base(original, cloner) { }
+    private RandomInitializer(bool deserializing) : base(deserializing) { }
+    private RandomInitializer(RandomInitializer original, Cloner cloner) : base(original, cloner) { }
     public RandomInitializer()
       : base() {
-      Parameters.Add(new ValueParameter<IntValue>("Seed", "The seed for the random number generator.", new IntValue(0)));
-      Parameters.Add(new ValueParameter<BoolValue>("SetSeedRandomly", "Whether the seed should be randomized for each call.", new BoolValue(true)));
+      Parameters.Add(new LookupParameter<IRandom>("Random", "The random number generator to use."));
     }
 
@@ -63,16 +48,12 @@
     }
 
-    public double[] Initialize(IClassificationProblemData data, int dimensions) {
-      var instances = data.TrainingIndices.Count();
+    public override double[,] Initialize(IClassificationProblemData data, Scaling scaling, int dimensions) {
       var attributes = data.AllowedInputVariables.Count();
 
-      var random = new MersenneTwister();
-      if (SetSeedRandomly) Seed = random.Next();
-      random.Reset(Seed);
-
-      var range = data.AllowedInputVariables.Select(x => data.Dataset.GetDoubleValues(x).Max() - data.Dataset.GetDoubleValues(x).Min()).ToArray();
-      var matrix = new double[attributes * dimensions];
-      for (int i = 0; i < matrix.Length; i++)
-        matrix[i] = random.NextDouble() / range[i / dimensions];
+      var random = RandomParameter.ActualValue;
+      var matrix = new double[attributes, dimensions];
+      for (int i = 0; i < attributes; i++)
+        for (int j = 0; j < dimensions; j++)
+          matrix[i, j] = random.NextDouble();
 
       return matrix;
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/INcaModelCreator.cs	(revision 9270)
@@ -0,0 +1,32 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaModelCreator : IOperator {
+
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/ModelCreation/NcaModelCreator.cs	(revision 9270)
@@ -0,0 +1,99 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaModelCreator", "Creates an NCA model with a given matrix.")]
+  [StorableClass]
+  public class NcaModelCreator : SingleSuccessorOperator, INcaModelCreator {
+
+    public ILookupParameter<IntValue> KParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["K"]; }
+    }
+
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrixGradients"]; }
+    }
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+
+    public ILookupParameter<Scaling> ScalingParameter {
+      get { return (ILookupParameter<Scaling>)Parameters["Scaling"]; }
+    }
+
+    public ILookupParameter<INcaModel> NcaModelParameter {
+      get { return (ILookupParameter<INcaModel>)Parameters["NcaModel"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaModelCreator(bool deserializing) : base(deserializing) { }
+    protected NcaModelCreator(NcaModelCreator original, Cloner cloner) : base(original, cloner) { }
+    public NcaModelCreator() {
+      Parameters.Add(new LookupParameter<IntValue>("K", "How many neighbors should be considered for classification."));
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The dimensions to which the feature space should be reduced to."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The optimized matrix."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrixGradients", "The gradients from the matrix that is being optimized."));
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<Scaling>("Scaling", "The scaling of the dataset."));
+      Parameters.Add(new LookupParameter<INcaModel>("NcaModel", "The NCA model that should be created."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaModelCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var k = KParameter.ActualValue.Value;
+      var dim = DimensionsParameter.ActualValue.Value;
+      var vector = NcaMatrixParameter.ActualValue;
+      var matrix = new double[vector.Length / dim, dim];
+
+      for (int i = 0; i < matrix.GetLength(0); i++)
+        for (int j = 0; j < dim; j++) {
+          matrix[i, j] = vector[i * dim + j];
+        }
+
+      var scaling = ScalingParameter.ActualValue;
+      var problemData = ProblemDataParameter.ActualValue;
+      NcaModelParameter.ActualValue = new NcaModel(k, matrix, problemData.Dataset, problemData.TrainingIndices, problemData.TargetVariable, problemData.AllowedInputVariables, scaling, problemData.ClassValues.ToArray());
+      return base.Apply();
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 9269)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaAlgorithm.cs	(revision 9270)
@@ -21,11 +21,10 @@
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Threading;
-using HeuristicLab.Analysis;
+using HeuristicLab.Algorithms.GradientDescent;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Operators;
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
@@ -33,7 +32,7 @@
 using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  internal delegate void Reporter(double quality, double[] coefficients, double[] gradients);
   /// <summary>
   /// Neighborhood Components Analysis
@@ -46,27 +45,73 @@
   [Creatable("Data Analysis")]
   [StorableClass]
-  public sealed class NcaAlgorithm : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+  public sealed class NcaAlgorithm : EngineAlgorithm {
+    #region Parameter Names
+    private const string SeedParameterName = "Seed";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string KParameterName = "K";
+    private const string DimensionsParameterName = "Dimensions";
+    private const string InitializationParameterName = "Initialization";
+    private const string NeighborSamplesParameterName = "NeighborSamples";
+    private const string IterationsParameterName = "Iterations";
+    private const string RegularizationParameterName = "Regularization";
+    private const string NcaModelCreatorParameterName = "NcaModelCreator";
+    private const string NcaSolutionCreatorParameterName = "NcaSolutionCreator";
+    private const string ApproximateGradientsParameterName = "ApproximateGradients";
+    private const string NcaMatrixParameterName = "NcaMatrix";
+    private const string NcaMatrixGradientsParameterName = "NcaMatrixGradients";
+    private const string QualityParameterName = "Quality";
+    #endregion
+
+    public override Type ProblemType { get { return typeof(IClassificationProblem); } }
+    public new IClassificationProblem Problem {
+      get { return (IClassificationProblem)base.Problem; }
+      set { base.Problem = value; }
+    }
+
     #region Parameter Properties
+    public IValueParameter<IntValue> SeedParameter {
+      get { return (IValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+    public IValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
     public IFixedValueParameter<IntValue> KParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["K"]; }
+      get { return (IFixedValueParameter<IntValue>)Parameters[KParameterName]; }
     }
     public IFixedValueParameter<IntValue> DimensionsParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["Dimensions"]; }
-    }
-    public IConstrainedValueParameter<INCAInitializer> InitializationParameter {
-      get { return (IConstrainedValueParameter<INCAInitializer>)Parameters["Initialization"]; }
+      get { return (IFixedValueParameter<IntValue>)Parameters[DimensionsParameterName]; }
+    }
+    public IConstrainedValueParameter<INcaInitializer> InitializationParameter {
+      get { return (IConstrainedValueParameter<INcaInitializer>)Parameters[InitializationParameterName]; }
     }
     public IFixedValueParameter<IntValue> NeighborSamplesParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["NeighborSamples"]; }
+      get { return (IFixedValueParameter<IntValue>)Parameters[NeighborSamplesParameterName]; }
     }
     public IFixedValueParameter<IntValue> IterationsParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters["Iterations"]; }
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
     }
     public IFixedValueParameter<DoubleValue> RegularizationParameter {
-      get { return (IFixedValueParameter<DoubleValue>)Parameters["Regularization"]; }
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[RegularizationParameterName]; }
+    }
+    public IValueParameter<BoolValue> ApproximateGradientsParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[ApproximateGradientsParameterName]; }
+    }
+    public IValueParameter<INcaModelCreator> NcaModelCreatorParameter {
+      get { return (IValueParameter<INcaModelCreator>)Parameters[NcaModelCreatorParameterName]; }
+    }
+    public IValueParameter<INcaSolutionCreator> NcaSolutionCreatorParameter {
+      get { return (IValueParameter<INcaSolutionCreator>)Parameters[NcaSolutionCreatorParameterName]; }
     }
     #endregion
 
     #region Properties
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
     public int K {
       get { return KParameter.Value.Value; }
@@ -88,4 +133,12 @@
       get { return RegularizationParameter.Value.Value; }
       set { RegularizationParameter.Value.Value = value; }
+    }
+    public INcaModelCreator NcaModelCreator {
+      get { return NcaModelCreatorParameter.Value; }
+      set { NcaModelCreatorParameter.Value = value; }
+    }
+    public INcaSolutionCreator NcaSolutionCreator {
+      get { return NcaSolutionCreatorParameter.Value; }
+      set { NcaSolutionCreatorParameter.Value = value; }
     }
     #endregion
@@ -96,18 +149,96 @@
     public NcaAlgorithm()
       : base() {
-      Parameters.Add(new FixedValueParameter<IntValue>("K", "The K for the nearest neighbor.", new IntValue(3)));
-      Parameters.Add(new FixedValueParameter<IntValue>("Dimensions", "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
-      Parameters.Add(new ConstrainedValueParameter<INCAInitializer>("Initialization", "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
-      Parameters.Add(new FixedValueParameter<IntValue>("NeighborSamples", "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one.", new IntValue(60)));
-      Parameters.Add(new FixedValueParameter<IntValue>("Iterations", "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(50)));
-      Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
-
-      INCAInitializer defaultInitializer = null;
-      foreach (var initializer in ApplicationManager.Manager.GetInstances<INCAInitializer>().OrderBy(x => x.ItemName)) {
-        if (initializer is LDAInitializer) defaultInitializer = initializer;
+      Parameters.Add(new ValueParameter<IntValue>(SeedParameterName, "The seed of the random number generator.", new IntValue(0)));
+      Parameters.Add(new ValueParameter<BoolValue>(SetSeedRandomlyParameterName, "A boolean flag that indicates whether the seed should be randomly reset each time the algorithm is run.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(KParameterName, "The K for the nearest neighbor.", new IntValue(3)));
+      Parameters.Add(new FixedValueParameter<IntValue>(DimensionsParameterName, "The number of dimensions that NCA should reduce the data to.", new IntValue(2)));
+      Parameters.Add(new ConstrainedValueParameter<INcaInitializer>(InitializationParameterName, "Which method should be used to initialize the matrix. Typically LDA (linear discriminant analysis) should provide a good estimate."));
+      Parameters.Add(new FixedValueParameter<IntValue>(NeighborSamplesParameterName, "How many of the neighbors should be sampled in order to speed up the calculation. This should be at least the value of k and at most the number of training instances minus one will be used.", new IntValue(60)));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "How many iterations the conjugate gradient (CG) method should be allowed to perform. The method might still terminate earlier if a local optima has already been reached.", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(RegularizationParameterName, "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
+      Parameters.Add(new ValueParameter<INcaModelCreator>(NcaModelCreatorParameterName, "Creates an NCA model out of the matrix.", new NcaModelCreator()));
+      Parameters.Add(new ValueParameter<INcaSolutionCreator>(NcaSolutionCreatorParameterName, "Creates an NCA solution given a model and some data.", new NcaSolutionCreator()));
+      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "True if the gradient should be approximated otherwise they are computed exactly.", new BoolValue()));
+
+      NcaSolutionCreatorParameter.Hidden = true;
+      ApproximateGradientsParameter.Hidden = true;
+
+      INcaInitializer defaultInitializer = null;
+      foreach (var initializer in ApplicationManager.Manager.GetInstances<INcaInitializer>().OrderBy(x => x.ItemName)) {
+        if (initializer is LdaInitializer) defaultInitializer = initializer;
         InitializationParameter.ValidValues.Add(initializer);
       }
       if (defaultInitializer != null) InitializationParameter.Value = defaultInitializer;
 
+      var randomCreator = new RandomCreator();
+      var ncaInitializer = new Placeholder();
+      var bfgsInitializer = new LbfgsInitializer();
+      var makeStep = new LbfgsMakeStep();
+      var branch = new ConditionalBranch();
+      var gradientCalculator = new NcaGradientCalculator();
+      var modelCreator = new Placeholder();
+      var updateResults = new LbfgsUpdateResults();
+      var analyzer = new LbfgsAnalyzer();
+      var finalModelCreator = new Placeholder();
+      var finalAnalyzer = new LbfgsAnalyzer();
+      var solutionCreator = new Placeholder();
+
+      OperatorGraph.InitialOperator = randomCreator;
+      randomCreator.SeedParameter.ActualName = SeedParameterName;
+      randomCreator.SeedParameter.Value = null;
+      randomCreator.SetSeedRandomlyParameter.ActualName = SetSeedRandomlyParameterName;
+      randomCreator.SetSeedRandomlyParameter.Value = null;
+      randomCreator.Successor = ncaInitializer;
+
+      ncaInitializer.Name = "(NcaInitializer)";
+      ncaInitializer.OperatorParameter.ActualName = InitializationParameterName;
+      ncaInitializer.Successor = bfgsInitializer;
+
+      bfgsInitializer.IterationsParameter.ActualName = IterationsParameterName;
+      bfgsInitializer.PointParameter.ActualName = NcaMatrixParameterName;
+      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      bfgsInitializer.Successor = makeStep;
+
+      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      makeStep.PointParameter.ActualName = NcaMatrixParameterName;
+      makeStep.Successor = branch;
+
+      branch.ConditionParameter.ActualName = makeStep.TerminationCriterionParameter.Name;
+      branch.FalseBranch = gradientCalculator;
+      branch.TrueBranch = finalModelCreator;
+
+      gradientCalculator.Successor = modelCreator;
+
+      modelCreator.OperatorParameter.ActualName = NcaModelCreatorParameterName;
+      modelCreator.Successor = updateResults;
+
+      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      updateResults.QualityParameter.ActualName = QualityParameterName;
+      updateResults.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
+      updateResults.Successor = analyzer;
+
+      analyzer.QualityParameter.ActualName = QualityParameterName;
+      analyzer.PointParameter.ActualName = NcaMatrixParameterName;
+      analyzer.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
+      analyzer.PointsTableParameter.ActualName = "Matrix table";
+      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
+      analyzer.QualitiesTableParameter.ActualName = "Qualities";
+      analyzer.Successor = makeStep;
+
+      finalModelCreator.OperatorParameter.ActualName = NcaModelCreatorParameterName;
+      finalModelCreator.Successor = finalAnalyzer;
+
+      finalAnalyzer.QualityParameter.ActualName = QualityParameterName;
+      finalAnalyzer.PointParameter.ActualName = NcaMatrixParameterName;
+      finalAnalyzer.QualityGradientsParameter.ActualName = NcaMatrixGradientsParameterName;
+      finalAnalyzer.PointsTableParameter.ActualName = analyzer.PointsTableParameter.ActualName;
+      finalAnalyzer.QualityGradientsTableParameter.ActualName = analyzer.QualityGradientsTableParameter.ActualName;
+      finalAnalyzer.QualitiesTableParameter.ActualName = analyzer.QualitiesTableParameter.ActualName;
+      finalAnalyzer.Successor = solutionCreator;
+
+      solutionCreator.OperatorParameter.ActualName = NcaSolutionCreatorParameterName;
+
       Problem = new ClassificationProblem();
     }
@@ -117,192 +248,7 @@
     }
 
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() {
-      if (!Parameters.ContainsKey("Regularization")) {
-        Parameters.Add(new FixedValueParameter<DoubleValue>("Regularization", "A non-negative paramter which can be set to increase generalization and avoid overfitting. If set to 0 the algorithm is similar to NCA as proposed by Goldberger et al.", new DoubleValue(0)));
-      }
-    }
-
     public override void Prepare() {
       if (Problem != null) base.Prepare();
     }
-
-    protected override void Run() {
-      var initializer = InitializationParameter.Value;
-
-      var clonedProblem = (IClassificationProblemData)Problem.ProblemData.Clone();
-      var model = Train(clonedProblem, K, Dimensions, NeighborSamples, Regularization, Iterations, initializer.Initialize(clonedProblem, Dimensions), ReportQuality, CancellationToken.None);
-      var solution = model.CreateClassificationSolution(clonedProblem);
-      if (!Results.ContainsKey("ClassificationSolution"))
-        Results.Add(new Result("ClassificationSolution", "The classification solution.", solution));
-      else Results["ClassificationSolution"].Value = solution;
-    }
-
-    public static INcaClassificationSolution CreateClassificationSolution(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
-      var clonedProblem = (IClassificationProblemData)data.Clone();
-      var model = Train(clonedProblem, k, dimensions, neighborSamples, regularization, iterations, initializer);
-      return model.CreateClassificationSolution(clonedProblem);
-    }
-
-    public static INcaModel Train(IClassificationProblemData problemData, int k, int dimensions, int neighborSamples, double regularization, int iterations, INCAInitializer initializer) {
-      return Train(problemData, k, dimensions, neighborSamples, regularization, iterations, initializer.Initialize(problemData, dimensions), null, CancellationToken.None);
-    }
-
-    public static INcaModel Train(IClassificationProblemData problemData, int k, int neighborSamples, double regularization, int iterations, double[,] initalMatrix) {
-      var matrix = new double[initalMatrix.Length];
-      for (int i = 0; i < initalMatrix.GetLength(0); i++)
-        for (int j = 0; j < initalMatrix.GetLength(1); j++)
-          matrix[i * initalMatrix.GetLength(1) + j] = initalMatrix[i, j];
-      return Train(problemData, k, initalMatrix.GetLength(1), neighborSamples, regularization, iterations, matrix, null, CancellationToken.None);
-    }
-
-    private static INcaModel Train(IClassificationProblemData data, int k, int dimensions, int neighborSamples, double regularization, int iterations, double[] matrix, Reporter reporter, CancellationToken cancellation) {
-      var scaling = new Scaling(data.Dataset, data.AllowedInputVariables, data.TrainingIndices);
-      var scaledData = AlglibUtil.PrepareAndScaleInputMatrix(data.Dataset, data.AllowedInputVariables, data.TrainingIndices, scaling);
-      var classes = data.Dataset.GetDoubleValues(data.TargetVariable, data.TrainingIndices).ToArray();
-      var attributes = scaledData.GetLength(1);
-
-      alglib.mincgstate state;
-      alglib.mincgreport rep;
-      alglib.mincgcreate(matrix, out state);
-      alglib.mincgsetcond(state, 0, 0, 0, iterations);
-      alglib.mincgsetxrep(state, true);
-      //alglib.mincgsetgradientcheck(state, 0.01);
-      int neighborSampleSize = neighborSamples;
-      Optimize(state, scaledData, classes, dimensions, neighborSampleSize, regularization, cancellation, reporter);
-      alglib.mincgresults(state, out matrix, out rep);
-      if (rep.terminationtype == -7) throw new InvalidOperationException("Gradient verification failed.");
-
-      var transformationMatrix = new double[attributes, dimensions];
-      var counter = 0;
-      for (var i = 0; i < attributes; i++)
-        for (var j = 0; j < dimensions; j++)
-          transformationMatrix[i, j] = matrix[counter++];
-
-      return new NcaModel(k, transformationMatrix, data.Dataset, data.TrainingIndices, data.TargetVariable, data.AllowedInputVariables, scaling, data.ClassValues.ToArray());
-    }
-
-    private static void Optimize(alglib.mincgstate state, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda, CancellationToken cancellation, Reporter reporter) {
-      while (alglib.mincgiteration(state)) {
-        if (cancellation.IsCancellationRequested) break;
-        if (state.needfg) {
-          Gradient(state.x, ref state.innerobj.f, state.innerobj.g, data, classes, dimensions, neighborSampleSize, lambda);
-          continue;
-        }
-        if (state.innerobj.xupdated) {
-          if (reporter != null)
-            reporter(state.innerobj.f, state.innerobj.x, state.innerobj.g);
-          continue;
-        }
-        throw new InvalidOperationException("Neighborhood Components Analysis: Error in Optimize() (some derivatives were not provided?)");
-      }
-    }
-
-    private static void Gradient(double[] A, ref double func, double[] grad, double[,] data, double[] classes, int dimensions, int neighborSampleSize, double lambda) {
-      var instances = data.GetLength(0);
-      var attributes = data.GetLength(1);
-
-      var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
-
-      alglib.sparsematrix probabilities;
-      alglib.sparsecreate(instances, instances, out probabilities);
-      var transformedDistances = new Dictionary<int, double>(instances);
-      for (int i = 0; i < instances; i++) {
-        var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
-        for (int k = 0; k < instances; k++) {
-          if (k == i) {
-            transformedDistances.Remove(k);
-            continue;
-          }
-          var kVector = new Matrix(GetRow(data, k));
-          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
-        }
-        var normalization = transformedDistances.Sum(x => x.Value);
-        if (normalization <= 0) continue;
-        foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSampleSize)) {
-          alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
-        }
-      }
-      alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
-
-      int t0 = 0, t1 = 0, r, c;
-      double val;
-      var pi = new double[instances];
-      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
-        if (classes[r].IsAlmost(classes[c])) {
-          pi[r] += val;
-        }
-      }
-
-      var innerSum = new double[attributes, attributes];
-      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
-        var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
-        vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
-
-        if (classes[r].IsAlmost(classes[c])) {
-          vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
-        }
-      }
-
-      func = -pi.Sum() + lambda * AMatrix.SumOfSquares();
-
-      r = 0;
-      var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
-      foreach (var g in newGrad) {
-        grad[r] = g + lambda * 2 * A[r];
-        r++;
-      }
-    }
-
-    private void ReportQuality(double func, double[] coefficients, double[] gradients) {
-      var instances = Problem.ProblemData.TrainingIndices.Count();
-      DataTable qualities;
-      if (!Results.ContainsKey("Optimization")) {
-        qualities = new DataTable("Optimization");
-        qualities.Rows.Add(new DataRow("Quality", string.Empty));
-        Results.Add(new Result("Optimization", qualities));
-      } else qualities = (DataTable)Results["Optimization"].Value;
-      qualities.Rows["Quality"].Values.Add(-func / instances);
-
-      string[] attributNames = Problem.ProblemData.AllowedInputVariables.ToArray();
-      if (gradients != null) {
-        DataTable grads;
-        if (!Results.ContainsKey("Gradients")) {
-          grads = new DataTable("Gradients");
-          for (int i = 0; i < gradients.Length; i++)
-            grads.Rows.Add(new DataRow(attributNames[i / Dimensions] + "-" + (i % Dimensions), string.Empty));
-          Results.Add(new Result("Gradients", grads));
-        } else grads = (DataTable)Results["Gradients"].Value;
-        for (int i = 0; i < gradients.Length; i++)
-          grads.Rows[attributNames[i / Dimensions] + "-" + (i % Dimensions)].Values.Add(gradients[i]);
-      }
-
-      if (!Results.ContainsKey("Quality")) {
-        Results.Add(new Result("Quality", new DoubleValue(-func / instances)));
-      } else ((DoubleValue)Results["Quality"].Value).Value = -func / instances;
-
-      var attributes = attributNames.Length;
-      var transformationMatrix = new double[attributes, Dimensions];
-      var counter = 0;
-      for (var i = 0; i < attributes; i++)
-        for (var j = 0; j < Dimensions; j++)
-          transformationMatrix[i, j] = coefficients[counter++];
-
-      var scaling = new Scaling(Problem.ProblemData.Dataset, attributNames, Problem.ProblemData.TrainingIndices);
-      var model = new NcaModel(K, transformationMatrix, Problem.ProblemData.Dataset, Problem.ProblemData.TrainingIndices, Problem.ProblemData.TargetVariable, attributNames, scaling, Problem.ProblemData.ClassValues.ToArray());
-
-      IClassificationSolution solution = model.CreateClassificationSolution(Problem.ProblemData);
-      if (!Results.ContainsKey("ClassificationSolution")) {
-        Results.Add(new Result("ClassificationSolution", solution));
-      } else {
-        Results["ClassificationSolution"].Value = solution;
-      }
-    }
-
-    private static IEnumerable<double> GetRow(double[,] data, int row) {
-      for (int i = 0; i < data.GetLength(1); i++)
-        yield return data[row, i];
-    }
-
   }
 }
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/NcaGradientCalculator.cs	(revision 9270)
@@ -0,0 +1,179 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaGradientCalculator", "Calculates the quality and gradient of a certain NCA matrix.")]
+  [StorableClass]
+  public class NcaGradientCalculator : SingleSuccessorOperator {
+
+    #region Parameter Properties
+    public ILookupParameter<IntValue> DimensionsParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["Dimensions"]; }
+    }
+
+    public ILookupParameter<IntValue> NeighborSamplesParameter {
+      get { return (ILookupParameter<IntValue>)Parameters["NeighborSamples"]; }
+    }
+
+    public ILookupParameter<DoubleValue> RegularizationParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["Regularization"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrix"]; }
+    }
+
+    public ILookupParameter<RealVector> NcaMatrixGradientsParameter {
+      get { return (ILookupParameter<RealVector>)Parameters["NcaMatrixGradients"]; }
+    }
+
+    public ILookupParameter<DoubleValue> QualityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
+    }
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+
+    public ILookupParameter<Scaling> ScalingParameter {
+      get { return (ILookupParameter<Scaling>)Parameters["Scaling"]; }
+    }
+    #endregion
+
+    [StorableConstructor]
+    protected NcaGradientCalculator(bool deserializing) : base(deserializing) { }
+    protected NcaGradientCalculator(NcaGradientCalculator original, Cloner cloner) : base(original, cloner) { }
+    public NcaGradientCalculator()
+      : base() {
+      Parameters.Add(new LookupParameter<IntValue>("Dimensions", "The dimensions to which the feature space should be reduced to."));
+      Parameters.Add(new LookupParameter<IntValue>("NeighborSamples", "The number of neighbors that should be taken into account at maximum."));
+      Parameters.Add(new LookupParameter<DoubleValue>("Regularization", "The regularization term that constrains the expansion of the projected space."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrix", "The optimized matrix."));
+      Parameters.Add(new LookupParameter<RealVector>("NcaMatrixGradients", "The gradients from the matrix that is being optimized."));
+      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The quality of the current matrix."));
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<Scaling>("Scaling", "The scaling of the dataset."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaGradientCalculator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+      var scaling = ScalingParameter.ActualValue;
+      var dimensions = DimensionsParameter.ActualValue.Value;
+      var neighborSamples = NeighborSamplesParameter.ActualValue.Value;
+      var regularization = RegularizationParameter.ActualValue.Value;
+
+      var vector = NcaMatrixParameter.ActualValue;
+      var gradients = NcaMatrixGradientsParameter.ActualValue;
+      if (gradients == null) {
+        gradients = new RealVector(vector.Length);
+        NcaMatrixGradientsParameter.ActualValue = gradients;
+      }
+
+      var data = AlglibUtil.PrepareAndScaleInputMatrix(problemData.Dataset, problemData.AllowedInputVariables,
+                                                       problemData.TrainingIndices, scaling);
+      var classes = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+      var quality = Gradient(vector, gradients, data, classes, dimensions, neighborSamples, regularization);
+      QualityParameter.ActualValue = new DoubleValue(quality);
+
+      return base.Apply();
+    }
+
+    private static double Gradient(RealVector A, RealVector grad, double[,] data, double[] classes, int dimensions, int neighborSamples, double regularization) {
+      var instances = data.GetLength(0);
+      var attributes = data.GetLength(1);
+
+      var AMatrix = new Matrix(A, A.Length / dimensions, dimensions);
+
+      alglib.sparsematrix probabilities;
+      alglib.sparsecreate(instances, instances, out probabilities);
+      var transformedDistances = new Dictionary<int, double>(instances);
+      for (int i = 0; i < instances; i++) {
+        var iVector = new Matrix(GetRow(data, i), data.GetLength(1));
+        for (int k = 0; k < instances; k++) {
+          if (k == i) {
+            transformedDistances.Remove(k);
+            continue;
+          }
+          var kVector = new Matrix(GetRow(data, k));
+          transformedDistances[k] = Math.Exp(-iVector.Multiply(AMatrix).Subtract(kVector.Multiply(AMatrix)).SumOfSquares());
+        }
+        var normalization = transformedDistances.Sum(x => x.Value);
+        if (normalization <= 0) continue;
+        foreach (var s in transformedDistances.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(neighborSamples)) {
+          alglib.sparseset(probabilities, i, s.Key, s.Value / normalization);
+        }
+      }
+      alglib.sparseconverttocrs(probabilities); // needed to enumerate in order (top-down and left-right)
+
+      int t0 = 0, t1 = 0, r, c;
+      double val;
+      var pi = new double[instances];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        if (classes[r].IsAlmost(classes[c])) {
+          pi[r] += val;
+        }
+      }
+
+      var innerSum = new double[attributes, attributes];
+      while (alglib.sparseenumerate(probabilities, ref t0, ref t1, out r, out c, out val)) {
+        var vector = new Matrix(GetRow(data, r)).Subtract(new Matrix(GetRow(data, c)));
+        vector.OuterProduct(vector).Multiply(val * pi[r]).AddTo(innerSum);
+
+        if (classes[r].IsAlmost(classes[c])) {
+          vector.OuterProduct(vector).Multiply(-val).AddTo(innerSum);
+        }
+      }
+
+      var func = -pi.Sum() + regularization * AMatrix.SumOfSquares();
+
+      r = 0;
+      var newGrad = AMatrix.Multiply(-2.0).Transpose().Multiply(new Matrix(innerSum)).Transpose();
+      foreach (var g in newGrad) {
+        grad[r] = g + regularization * 2 * A[r];
+        r++;
+      }
+
+      return func;
+    }
+
+    private static IEnumerable<double> GetRow(double[,] data, int row) {
+      for (int i = 0; i < data.GetLength(1); i++)
+        yield return data[row, i];
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/INcaSolutionCreator.cs	(revision 9270)
@@ -0,0 +1,28 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Core;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  public interface INcaSolutionCreator : IOperator {
+
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 9270)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Nca/SolutionCreation/NcaSolutionCreator.cs	(revision 9270)
@@ -0,0 +1,91 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Operators;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  [Item("NcaSolutionCreator", "Creates an NCA solution with a given model and some given data.")]
+  [StorableClass]
+  public class NcaSolutionCreator : SingleSuccessorOperator, INcaSolutionCreator {
+
+    public ILookupParameter<IClassificationProblemData> ProblemDataParameter {
+      get { return (ILookupParameter<IClassificationProblemData>)Parameters["ProblemData"]; }
+    }
+
+    public ILookupParameter<INcaModel> NcaModelParameter {
+      get { return (ILookupParameter<INcaModel>)Parameters["NcaModel"]; }
+    }
+
+    public ILookupParameter<INcaClassificationSolution> NcaSolutionParameter {
+      get { return (ILookupParameter<INcaClassificationSolution>)Parameters["NcaSolution"]; }
+    }
+
+    public ILookupParameter<ResultCollection> ResultsParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters["Results"]; }
+    }
+
+    [StorableConstructor]
+    protected NcaSolutionCreator(bool deserializing) : base(deserializing) { }
+    protected NcaSolutionCreator(NcaSolutionCreator original, Cloner cloner) : base(original, cloner) { }
+    public NcaSolutionCreator()
+      : base() {
+      Parameters.Add(new LookupParameter<IClassificationProblemData>("ProblemData", "The classification problem data."));
+      Parameters.Add(new LookupParameter<INcaModel>("NcaModel", "The NCA model that should be created."));
+      Parameters.Add(new LookupParameter<INcaClassificationSolution>("NcaSolution", "The created NCA solution."));
+      Parameters.Add(new LookupParameter<ResultCollection>("Results", "The results collection to store the results."));
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NcaSolutionCreator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var problemData = ProblemDataParameter.ActualValue;
+      var model = NcaModelParameter.ActualValue;
+      var results = ResultsParameter.ActualValue;
+
+      var s = model.CreateClassificationSolution(problemData);
+      NcaSolutionParameter.ActualValue = s;
+
+      if (!results.ContainsKey("Solution")) {
+        results.Add(new Result("Solution", "The NCA classification solution", s));
+        results.Add(new Result("Accuracy (training)",
+                               "The accuracy of the NCA solution on the training partition.",
+                               new DoubleValue(s.TrainingAccuracy)));
+        results.Add(new Result("Accuracy (test)",
+                               "The accuracy of the NCA solution on the test partition.",
+                               new DoubleValue(s.TestAccuracy)));
+      } else {
+        results["Solution"].Value = s;
+        results["Accuracy (training)"].Value = new DoubleValue(s.TrainingAccuracy);
+        results["Accuracy (test)"].Value = new DoubleValue(s.TestAccuracy);
+      }
+      return base.Apply();
+    }
+  }
+}
