Changeset 14807 for branches/TSNE/HeuristicLab.Algorithms.DataAnalysis

Timestamp:

03/30/17 19:06:44 (8 years ago)

Author:

gkronber

Message:

#2700: support clone, persistence and pause/resume

Location:

branches/TSNE/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE

Files:

• TSNEAlgorithm.cs (modified) (11 diffs)
• TSNEStatic.cs (modified) (5 diffs)

branches/TSNE/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEAlgorithm.cs

@@ -46 +46 @@
   public sealed class TSNEAlgorithm : BasicAlgorithm {
     public override bool SupportsPause {
-      get { return false; }
+      get { return true; }
     }
     public override Type ProblemType {
@@ -182 +182 @@
       set { NormalizationParameter.Value.Value = value; }
     }
-    [Storable]
-    public TSNE<double[]> tsne;
     #endregion
 
@@ -189 +187 @@
     [StorableConstructor]
     private TSNEAlgorithm(bool deserializing) : base(deserializing) { }
-    private TSNEAlgorithm(TSNEAlgorithm original, Cloner cloner) : base(original, cloner) { }
+
+    private TSNEAlgorithm(TSNEAlgorithm original, Cloner cloner) : base(original, cloner) {
+      this.dataRowNames = new Dictionary<string, List<int>>(original.dataRowNames);
+      this.dataRows = original.dataRows.ToDictionary(kvp => kvp.Key, kvp => cloner.Clone(kvp.Value));
+      if(original.state != null)
+        this.state = cloner.Clone(original.state);
+      this.iter = original.iter;
+    }
     public override IDeepCloneable Clone(Cloner cloner) { return new TSNEAlgorithm(this, cloner); }
     public TSNEAlgorithm() {
       Problem = new RegressionProblem();
       Parameters.Add(new ValueParameter<IDistance<double[]>>(DistanceParameterName, "The distance function used to differentiate similar from non-similar points", new EuclideanDistance()));
-      Parameters.Add(new FixedValueParameter<DoubleValue>(PerplexityParameterName, "Perplexity-Parameter of tSNE. Comparable to k in a k-nearest neighbour algorithm. Recommended value is floor(number of points /3) or lower", new DoubleValue(25)));
-      Parameters.Add(new FixedValueParameter<DoubleValue>(ThetaParameterName, "Value describing how much appoximated gradients my differ from exact gradients. Set to 0 for exact calculation and in [0,1] otherwise \n CAUTION: exact calculation of forces requires building a non-sparse N*N matrix where N is the number of data points\n This may exceed memory limitations", new DoubleValue(0)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PerplexityParameterName, "Perplexity-parameter of tSNE. Comparable to k in a k-nearest neighbour algorithm. Recommended value is floor(number of points /3) or lower", new DoubleValue(25)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(ThetaParameterName, "Value describing how much appoximated gradients my differ from exact gradients. Set to 0 for exact calculation and in [0,1] otherwise. CAUTION: exact calculation of forces requires building a non-sparse N*N matrix where N is the number of data points. This may exceed memory limitations.", new DoubleValue(0)));
       Parameters.Add(new FixedValueParameter<IntValue>(NewDimensionsParameterName, "Dimensionality of projected space (usually 2 for easy visual analysis)", new IntValue(2)));
-      Parameters.Add(new FixedValueParameter<IntValue>(MaxIterationsParameterName, "Maximum number of iterations for gradient descent", new IntValue(1000)));
-      Parameters.Add(new FixedValueParameter<IntValue>(StopLyingIterationParameterName, "Number of iterations after which p is no longer approximated", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<IntValue>(MomentumSwitchIterationParameterName, "Number of iterations after which the momentum in the gradient descent is switched", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<DoubleValue>(InitialMomentumParameterName, "The initial momentum in the gradient descent", new DoubleValue(0.5)));
-      Parameters.Add(new FixedValueParameter<DoubleValue>(FinalMomentumParameterName, "The final momentum", new DoubleValue(0.8)));
-      Parameters.Add(new FixedValueParameter<DoubleValue>(EtaParameterName, "Gradient descent learning rate", new DoubleValue(200)));
-      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "If the seed should be random", new BoolValue(true)));
-      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The seed used if it should not be random", new IntValue(0)));
-      Parameters.Add(new FixedValueParameter<StringValue>(ClassesParameterName, "name of the column specifying the class lables of each data point. \n if the lable column can not be found training/test is used as labels", new StringValue("none")));
-      Parameters.Add(new FixedValueParameter<BoolValue>(NormalizationParameterName, "Whether the data should be zero centered and have variance of 1 for each variable, so different scalings are ignored", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaxIterationsParameterName, "Maximum number of iterations for gradient descent.", new IntValue(1000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(StopLyingIterationParameterName, "Number of iterations after which p is no longer approximated.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MomentumSwitchIterationParameterName, "Number of iterations after which the momentum in the gradient descent is switched.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(InitialMomentumParameterName, "The initial momentum in the gradient descent.", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(FinalMomentumParameterName, "The final momentum.", new DoubleValue(0.8)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(EtaParameterName, "Gradient descent learning rate.", new DoubleValue(200)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "If the seed should be random.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The seed used if it should not be random.", new IntValue(0)));
+      Parameters.Add(new FixedValueParameter<StringValue>(ClassesParameterName, "name of the column specifying the class lables of each data point. If the label column can not be found training/test is used as labels.", new StringValue("none")));
+      Parameters.Add(new FixedValueParameter<BoolValue>(NormalizationParameterName, "Whether the data should be zero centered and have variance of 1 for each variable, so different scalings are ignored.", new BoolValue(true)));
 
       MomentumSwitchIterationParameter.Hidden = true;
@@ -217 +222 @@
 
     [Storable]
-    private Dictionary<string, List<int>> dataRowNames;    // TODO
+    private Dictionary<string, List<int>> dataRowNames;
     [Storable]
-    private Dictionary<string, ScatterPlotDataRow> dataRows; // TODO
-
+    private Dictionary<string, ScatterPlotDataRow> dataRows;
+    [Storable]
+    private TSNEStatic<double[]>.TSNEState state;
+    [Storable]
+    private int iter;
+
+    public override void Prepare() {
+      base.Prepare();
+      dataRowNames = null;
+      dataRows = null;
+      state = null;
+    }
 
     protected override void Run(CancellationToken cancellationToken) {
       var problemData = Problem.ProblemData;
-
-      // set up and run tSNE
-      if (SetSeedRandomly) Seed = new System.Random().Next();
-      var random = new MersenneTwister((uint)Seed);
-      var dataset = problemData.Dataset;
-      var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
-      var data = new double[dataset.Rows][];
-      for (var row = 0; row < dataset.Rows; row++) data[row] = allowedInputVariables.Select(col => dataset.GetDoubleValue(col, row)).ToArray();
-      if (Normalization) data = NormalizeData(data);
-
-      var tsneState = TSNE<double[]>.CreateState(data, Distance, random, NewDimensions, Perplexity, Theta, StopLyingIteration, MomentumSwitchIteration, InitialMomentum, FinalMomentum, Eta);
-
-      SetUpResults(data);
-      for (int iter = 0; iter < MaxIterations && !cancellationToken.IsCancellationRequested; iter++)
-      {
-        TSNE<double[]>.Iterate(tsneState);
-        Analyze(tsneState);
+      // set up and initialized everything if necessary
+      if(state == null) {
+        if(SetSeedRandomly) Seed = new System.Random().Next();
+        var random = new MersenneTwister((uint)Seed);
+        var dataset = problemData.Dataset;
+        var allowedInputVariables = problemData.AllowedInputVariables.ToArray();
+        var data = new double[dataset.Rows][];
+        for(var row = 0; row < dataset.Rows; row++)
+          data[row] = allowedInputVariables.Select(col => dataset.GetDoubleValue(col, row)).ToArray();
+
+        if(Normalization) data = NormalizeData(data);
+
+        state = TSNEStatic<double[]>.CreateState(data, Distance, random, NewDimensions, Perplexity, Theta,
+          StopLyingIteration, MomentumSwitchIteration, InitialMomentum, FinalMomentum, Eta);
+
+        SetUpResults(data);
+        iter = 0;
+      }
+      for(; iter < MaxIterations && !cancellationToken.IsCancellationRequested; iter++) {
+        TSNEStatic<double[]>.Iterate(state);
+        Analyze(state);
       }
     }
 
     private void SetUpResults(IReadOnlyCollection<double[]> data) {
-      if (Results == null) return;
+      if(Results == null) return;
       var results = Results;
       dataRowNames = new Dictionary<string, List<int>>();
@@ -252 +271 @@
 
       //color datapoints acording to classes variable (be it double or string)
-      if (problemData.Dataset.VariableNames.Contains(Classes)) {
-        if ((problemData.Dataset as Dataset).VariableHasType<string>(Classes)) {
+      if(problemData.Dataset.VariableNames.Contains(Classes)) {
+        if((problemData.Dataset as Dataset).VariableHasType<string>(Classes)) {
           var classes = problemData.Dataset.GetStringValues(Classes).ToArray();
-          for (var i = 0; i < classes.Length; i++) {
-            if (!dataRowNames.ContainsKey(classes[i])) dataRowNames.Add(classes[i], new List<int>());
+          for(var i = 0; i < classes.Length; i++) {
+            if(!dataRowNames.ContainsKey(classes[i])) dataRowNames.Add(classes[i], new List<int>());
             dataRowNames[classes[i]].Add(i);
           }
-        } else if ((problemData.Dataset as Dataset).VariableHasType<double>(Classes)) {
+        } else if((problemData.Dataset as Dataset).VariableHasType<double>(Classes)) {
           var classValues = problemData.Dataset.GetDoubleValues(Classes).ToArray();
           var max = classValues.Max() + 0.1;     // TODO consts
           var min = classValues.Min() - 0.1;
           const int contours = 8;
-          for (var i = 0; i < contours; i++) {
+          for(var i = 0; i < contours; i++) {
             var contourname = GetContourName(i, min, max, contours);
             dataRowNames.Add(contourname, new List<int>());
@@ -271 +290 @@
             dataRows[contourname].VisualProperties.PointSize = i + 3;
           }
-          for (var i = 0; i < classValues.Length; i++) {
+          for(var i = 0; i < classValues.Length; i++) {
             dataRowNames[GetContourName(classValues[i], min, max, contours)].Add(i);
           }
@@ -280 +299 @@
       }
 
-      if (!results.ContainsKey(IterationResultName)) results.Add(new Result(IterationResultName, new IntValue(0)));
+      if(!results.ContainsKey(IterationResultName)) results.Add(new Result(IterationResultName, new IntValue(0)));
       else ((IntValue)results[IterationResultName].Value).Value = 0;
 
-      if (!results.ContainsKey(ErrorResultName)) results.Add(new Result(ErrorResultName, new DoubleValue(0)));
+      if(!results.ContainsKey(ErrorResultName)) results.Add(new Result(ErrorResultName, new DoubleValue(0)));
       else ((DoubleValue)results[ErrorResultName].Value).Value = 0;
 
-      if (!results.ContainsKey(ErrorPlotResultName)) results.Add(new Result(ErrorPlotResultName, new DataTable(ErrorPlotResultName, "Development of errors during gradient descent")));
+      if(!results.ContainsKey(ErrorPlotResultName)) results.Add(new Result(ErrorPlotResultName, new DataTable(ErrorPlotResultName, "Development of errors during gradient descent")));
       else results[ErrorPlotResultName].Value = new DataTable(ErrorPlotResultName, "Development of errors during gradient descent");
 
       var plot = results[ErrorPlotResultName].Value as DataTable;
-      if (plot == null) throw new ArgumentException("could not create/access error data table in results collection");
-
-      if (!plot.Rows.ContainsKey("errors")) plot.Rows.Add(new DataRow("errors"));
+      if(plot == null) throw new ArgumentException("could not create/access error data table in results collection");
+
+      if(!plot.Rows.ContainsKey("errors")) plot.Rows.Add(new DataRow("errors"));
       plot.Rows["errors"].Values.Clear();
 
@@ -299 +318 @@
     }
 
-    private void Analyze(TSNE<double[]>.TSNEState tsneState) {
-      if (Results == null) return;
+    private void Analyze(TSNEStatic<double[]>.TSNEState tsneState) {
+      if(Results == null) return;
       var results = Results;
       var plot = results[ErrorPlotResultName].Value as DataTable;
-      if (plot == null) throw new ArgumentException("Could not create/access error data table in results collection.");
+      if(plot == null) throw new ArgumentException("Could not create/access error data table in results collection.");
       var errors = plot.Rows["errors"].Values;
       var c = tsneState.EvaluateError();
       errors.Add(c);
-      ((IntValue)results[IterationResultName].Value).Value = tsneState.iter + 1;
+      ((IntValue)results[IterationResultName].Value).Value = tsneState.iter;
       ((DoubleValue)results[ErrorResultName].Value).Value = errors.Last();
 
@@ -317 +336 @@
 
     private void FillScatterPlot(double[,] lowDimData, ScatterPlot plot) {
-      foreach (var rowName in dataRowNames.Keys) {
-        if (!plot.Rows.ContainsKey(rowName))
+      foreach(var rowName in dataRowNames.Keys) {
+        if(!plot.Rows.ContainsKey(rowName))
           plot.Rows.Add(dataRows.ContainsKey(rowName) ? dataRows[rowName] : new ScatterPlotDataRow(rowName, "", new List<Point2D<double>>()));
         plot.Rows[rowName].Points.Replace(dataRowNames[rowName].Select(i => new Point2D<double>(lowDimData[i, 0], lowDimData[i, 1])));
@@ -328 +347 @@
       var min = new double[data.GetLength(1)];
       var res = new double[data.GetLength(0), data.GetLength(1)];
-      for (var i = 0; i < max.Length; i++) max[i] = min[i] = data[0, i];
-      for (var i = 0; i < data.GetLength(0); i++)
-        for (var j = 0; j < data.GetLength(1); j++) {
+      for(var i = 0; i < max.Length; i++) max[i] = min[i] = data[0, i];
+      for(var i = 0; i < data.GetLength(0); i++)
+        for(var j = 0; j < data.GetLength(1); j++) {
           var v = data[i, j];
           max[j] = Math.Max(max[j], v);
           min[j] = Math.Min(min[j], v);
         }
-      for (var i = 0; i < data.GetLength(0); i++) {
-        for (var j = 0; j < data.GetLength(1); j++) {
+      for(var i = 0; i < data.GetLength(0); i++) {
+        for(var j = 0; j < data.GetLength(1); j++) {
           res[i, j] = (data[i, j] - (max[j] + min[j]) / 2) / (max[j] - min[j]);
         }
@@ -348 +367 @@
       var sd = new double[n];
       var nData = new double[data.Count][];
-      for (var i = 0; i < n; i++) {
+      for(var i = 0; i < n; i++) {
         var i1 = i;
         sd[i] = Enumerable.Range(0, data.Count).Select(x => data[x][i1]).StandardDeviation();
         mean[i] = Enumerable.Range(0, data.Count).Select(x => data[x][i1]).Average();
       }
-      for (var i = 0; i < data.Count; i++) {
+      for(var i = 0; i < data.Count; i++) {
         nData[i] = new double[n];
-        for (var j = 0; j < n; j++) nData[i][j] = (data[i][j] - mean[j]) / sd[j];
+        for(var j = 0; j < n; j++) nData[i][j] = (data[i][j] - mean[j]) / sd[j];
       }
       return nData;

branches/TSNE/HeuristicLab.Algorithms.DataAnalysis/3.4/TSNE/TSNEStatic.cs

@@ -60 +60 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Optimization;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Random;
@@ -65 +66 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  public class TSNE<T> {
+  public class TSNEStatic<T> {
 
     [StorableClass]
@@ -166 +167 @@
       }
 
+      [StorableConstructor]
+      public TSNEState(bool deserializing)  { }
       public TSNEState(T[] data, IDistance<T> distance, IRandom random, int newDimensions, double perplexity, double theta, int stopLyingIter, int momSwitchIter, double momentum, double finalMomentum, double eta) {
         this.distance = distance;
@@ -525 +528 @@
         for(var i = 0; i < noElem; i++) symValP[i] /= 2.0;
       }
-
     }
 
-    public static TSNEState CreateState(T[] data, IDistance<T> distance, IRandom random, int newDimensions = 2, double perplexity = 25, double theta = 0,
-      int stopLyingIter = 250, int momSwitchIter = 250, double momentum = .5, double finalMomentum = .8, double eta = 200.0
+    /// <summary>
+    /// Simple interface to tSNE
+    /// </summary>
+    /// <param name="data"></param>
+    /// <param name="distance">The distance function used to differentiate similar from non-similar points, e.g. Euclidean distance.</param>
+    /// <param name="random">Random number generator</param>
+    /// <param name="newDimensions">Dimensionality of projected space (usually 2 for easy visual analysis).</param>
+    /// <param name="perplexity">Perplexity parameter of tSNE. Comparable to k in a k-nearest neighbour algorithm. Recommended value is floor(number of points /3) or lower</param>
+    /// <param name="iterations">Maximum number of iterations for gradient descent.</param>
+    /// <param name="theta">Value describing how much appoximated gradients my differ from exact gradients. Set to 0 for exact calculation and in [0,1] otherwise. CAUTION: exact calculation of forces requires building a non-sparse N*N matrix where N is the number of data points. This may exceed memory limitations.</param>
+    /// <param name="stopLyingIter">Number of iterations after which p is no longer approximated.</param>
+    /// <param name="momSwitchIter">Number of iterations after which the momentum in the gradient descent is switched.</param>
+    /// <param name="momentum">The initial momentum in the gradient descent.</param>
+    /// <param name="finalMomentum">The final momentum in gradient descent (after momentum switch).</param>
+    /// <param name="eta">Gradient descent learning rate.</param>
+    /// <returns></returns>
+    public static double[,] Run(T[] data, IDistance<T> distance, IRandom random,
+      int newDimensions = 2, double perplexity = 25, int iterations = 1000,
+      double theta = 0,
+      int stopLyingIter = 250, int momSwitchIter = 250, double momentum = .5,
+      double finalMomentum = .8, double eta = 200.0
+      ) {
+      var state = CreateState(data, distance, random, newDimensions, perplexity,
+        theta, stopLyingIter, momSwitchIter, momentum, finalMomentum, eta);
+
+      for(int i = 0; i < iterations - 1; i++) {
+        Iterate(state);
+      }
+      return Iterate(state);
+    }
+
+    public static TSNEState CreateState(T[] data, IDistance<T> distance, IRandom random,
+      int newDimensions = 2, double perplexity = 25, double theta = 0,
+      int stopLyingIter = 250, int momSwitchIter = 250, double momentum = .5,
+      double finalMomentum = .8, double eta = 200.0
       ) {
       return new TSNEState(data, distance, random, newDimensions, perplexity, theta, stopLyingIter, momSwitchIter, momentum, finalMomentum, eta);
@@ -564 +599 @@
       // Make solution zero-mean
       ZeroMean(state.newData);
+
       // Stop lying about the P-values after a while, and switch momentum
-
       if(state.iter == state.stopLyingIter) {
         if(state.exact)
-          for(var i = 0; i < state.noDatapoints; i++) for(var j = 0; j < state.noDatapoints; j++) state.p[i, j] /= 12.0;                                   //XXX why 12?
+          for(var i = 0; i < state.noDatapoints; i++)
+            for(var j = 0; j < state.noDatapoints; j++)
+              state.p[i, j] /= 12.0;                                   //XXX why 12?
         else
-          for(var i = 0; i < state.rowP[state.noDatapoints]; i++) state.valP[i] /= 12.0;                       // XXX are we not scaling all values? 
+          for(var i = 0; i < state.rowP[state.noDatapoints]; i++)
+            state.valP[i] /= 12.0;                       // XXX are we not scaling all values? 
       }