Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Dataset.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Dataset.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Dataset.cs	(revision 8085)
@@ -131,6 +131,6 @@
         for (int col = 0; col < storableData.GetLength(1); col++) {
           string columName = variableNames[col];
-          var values = new List<double>();
-          for (int row = 0; row < storableData.GetLength(0); row++) {
+          var values = new List<double>(rows);
+          for (int row = 0; row < rows; row++) {
             values.Add(storableData[row, col]);
           }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj	(revision 8085)
@@ -141,4 +141,5 @@
     <Compile Include="Interfaces\Regression\IRegressionEnsembleSolution.cs" />
     <Compile Include="Implementation\Regression\RegressionSolutionBase.cs" />
+    <Compile Include="OnlineCalculators\HoeffdingsDependenceCalculator.cs" />
     <Compile Include="OnlineCalculators\OnlineMaxAbsoluteErrorCalculator.cs" />
     <Compile Include="OnlineCalculators\OnlineMeanErrorCalculator.cs" />
@@ -180,5 +181,4 @@
     <Compile Include="Implementation\Regression\RegressionSolution.cs" />
     <Compile Include="Plugin.cs" />
-    <Compile Include="TableFileParser.cs" />
     <Compile Include="Implementation\Classification\ThresholdCalculators\AccuracyMaximizationThresholdCalculator.cs" />
     <Compile Include="Implementation\Classification\ThresholdCalculators\NormalDistributionCutPointsThresholdCalculator.cs" />
@@ -190,51 +190,4 @@
     <Compile Include="Interfaces\IDataAnalysisProblem.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
-      <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>
-      <Name>HeuristicLab.Collections-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Common.Resources\3.3\HeuristicLab.Common.Resources-3.3.csproj">
-      <Project>{0E27A536-1C4A-4624-A65E-DC4F4F23E3E1}</Project>
-      <Name>HeuristicLab.Common.Resources-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
-      <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
-      <Name>HeuristicLab.Common-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
-      <Project>{C36BD924-A541-4A00-AFA8-41701378DDC5}</Project>
-      <Name>HeuristicLab.Core-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
-      <Project>{BBAB9DF5-5EF3-4BA8-ADE9-B36E82114937}</Project>
-      <Name>HeuristicLab.Data-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
-      <Project>{14AB8D24-25BC-400C-A846-4627AA945192}</Project>
-      <Name>HeuristicLab.Optimization-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
-      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
-      <Name>HeuristicLab.Parameters-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
-      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
-      <Name>HeuristicLab.Persistence-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
-      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
-      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
@@ -255,5 +208,56 @@
     </BootstrapperPackage>
   </ItemGroup>
-  <ItemGroup />
+  <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
+      <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>
+      <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common.Resources\3.3\HeuristicLab.Common.Resources-3.3.csproj">
+      <Project>{0E27A536-1C4A-4624-A65E-DC4F4F23E3E1}</Project>
+      <Name>HeuristicLab.Common.Resources-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
+      <Project>{A9AD58B9-3EF9-4CC1-97E5-8D909039FF5C}</Project>
+      <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
+      <Project>{C36BD924-A541-4A00-AFA8-41701378DDC5}</Project>
+      <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
+      <Project>{BBAB9DF5-5EF3-4BA8-ADE9-B36E82114937}</Project>
+      <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
+      <Project>{14AB8D24-25BC-400C-A846-4627AA945192}</Project>
+      <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
+      <Project>{56F9106A-079F-4C61-92F6-86A84C2D84B7}</Project>
+      <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
+      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
+      <Name>HeuristicLab.Persistence-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Problems.Instances\3.3\HeuristicLab.Problems.Instances-3.3.csproj">
+      <Project>{3540E29E-4793-49E7-8EE2-FEA7F61C3994}</Project>
+      <Name>HeuristicLab.Problems.Instances-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblem.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblem.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblem.cs	(revision 8085)
@@ -40,9 +40,4 @@
       ProblemData = new ClassificationProblemData();
     }
-
-    public override void ImportProblemDataFromFile(string fileName) {
-      ClassificationProblemData problemData = ClassificationProblemData.ImportFromFile(fileName);
-      ProblemData = problemData;
-    }
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs	(revision 8085)
@@ -22,5 +22,4 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using HeuristicLab.Common;
@@ -403,18 +402,4 @@
     }
     #endregion
-
-    #region Import from file
-    public static ClassificationProblemData ImportFromFile(string fileName) {
-      TableFileParser csvFileParser = new TableFileParser();
-      csvFileParser.Parse(fileName);
-
-      Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      dataset.Name = Path.GetFileName(fileName);
-
-      ClassificationProblemData problemData = new ClassificationProblemData(dataset, dataset.DoubleVariables.Skip(1), dataset.DoubleVariables.First());
-      problemData.Name = "Data imported from " + Path.GetFileName(fileName);
-      return problemData;
-    }
-    #endregion
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblem.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblem.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblem.cs	(revision 8085)
@@ -38,9 +38,4 @@
       ProblemData = new ClusteringProblemData();
     }
-
-    public override void ImportProblemDataFromFile(string fileName) {
-      ClusteringProblemData problemData = ClusteringProblemData.ImportFromFile(fileName);
-      ProblemData = problemData;
-    }
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblemData.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblemData.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblemData.cs	(revision 8085)
@@ -21,5 +21,4 @@
 
 using System.Collections.Generic;
-using System.IO;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -90,19 +89,4 @@
       : base(dataset, allowedInputVariables) {
     }
-
-
-    #region Import from file
-    public static ClusteringProblemData ImportFromFile(string fileName) {
-      TableFileParser csvFileParser = new TableFileParser();
-      csvFileParser.Parse(fileName);
-
-      Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      dataset.Name = Path.GetFileName(fileName);
-
-      ClusteringProblemData problemData = new ClusteringProblemData(dataset, dataset.DoubleVariables);
-      problemData.Name = "Data imported from " + Path.GetFileName(fileName);
-      return problemData;
-    }
-    #endregion
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblem.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblem.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblem.cs	(revision 8085)
@@ -26,9 +26,10 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.Instances;
 
 namespace HeuristicLab.Problems.DataAnalysis {
   [StorableClass]
-  public abstract class DataAnalysisProblem<T> : Problem,
-    IDataAnalysisProblem<T>
+  public abstract class DataAnalysisProblem<T> : Problem, IDataAnalysisProblem<T>,
+    IProblemInstanceConsumer<T>, IProblemInstanceExporter<T>
     where T : class, IDataAnalysisProblemData {
     private const string ProblemDataParameterName = "ProblemData";
@@ -91,5 +92,16 @@
     }
 
-    public abstract void ImportProblemDataFromFile(string fileName);
+    #region Import & Export
+    public void Load(T data) {
+      Name = data.Name;
+      Description = data.Description;
+      ProblemData = data;
+      OnReset();
+    }
+
+    public T Export() {
+      return ProblemData;
+    }
+    #endregion
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblem.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblem.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblem.cs	(revision 8085)
@@ -40,9 +40,4 @@
       ProblemData = new RegressionProblemData();
     }
-
-    public override void ImportProblemDataFromFile(string fileName) {
-      RegressionProblemData problemData = RegressionProblemData.ImportFromFile(fileName);
-      ProblemData = problemData;
-    }
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs	(revision 8085)
@@ -22,5 +22,4 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using HeuristicLab.Common;
@@ -136,18 +135,4 @@
       OnChanged();
     }
-
-    #region Import from file
-    public static RegressionProblemData ImportFromFile(string fileName) {
-      TableFileParser csvFileParser = new TableFileParser();
-      csvFileParser.Parse(fileName);
-
-      Dataset dataset = new Dataset(csvFileParser.VariableNames, csvFileParser.Values);
-      dataset.Name = Path.GetFileName(fileName);
-
-      RegressionProblemData problemData = new RegressionProblemData(dataset, dataset.DoubleVariables.Skip(1), dataset.DoubleVariables.First());
-      problemData.Name = "Data imported from " + Path.GetFileName(fileName);
-      return problemData;
-    }
-    #endregion
   }
 }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisProblem.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisProblem.cs	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisProblem.cs	(revision 8085)
@@ -29,6 +29,4 @@
     IDataAnalysisProblemData ProblemData { get; }
     event EventHandler ProblemDataChanged;
-
-    void ImportProblemDataFromFile(string fileName);
   }
 
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/HoeffdingsDependenceCalculator.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/HoeffdingsDependenceCalculator.cs	(revision 8085)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/HoeffdingsDependenceCalculator.cs	(revision 8085)
@@ -0,0 +1,130 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Problems.DataAnalysis {
+  public class HoeffdingsDependenceCalculator {
+
+    public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
+      double d = HoeffD(originalValues, estimatedValues, out errorState);
+      if (errorState != OnlineCalculatorError.None) return double.NaN;
+      return d;
+    }
+
+    /// <summary>
+    /// computes Hoeffding's dependence coefficient. 
+    /// Source: hoeffd.r from R package hmisc http://cran.r-project.org/web/packages/Hmisc/index.html
+    /// </summary>
+    private static double HoeffD(IEnumerable<double> xs, IEnumerable<double> ys, out OnlineCalculatorError errorState) {
+      double[] rx = TiedRank(xs);
+      double[] ry = TiedRank(ys);
+      if (rx.Length != ry.Length) throw new ArgumentException("The number of elements in xs and ys does not match");
+      double[] rxy = TiedRank(xs, ys);
+
+      int n = rx.Length;
+      double q = 0, r = 0, s = 0;
+      double scaling = 1.0 / (n * (n - 1));
+      for (int i = 0; i < n; i++) {
+        q += (rx[i] - 1) * (rx[i] - 2) * (ry[i] - 1) * (ry[i] - 2) * scaling;
+        r += (rx[i] - 2) * (ry[i] - 2) * rxy[i] * scaling;
+        s += rxy[i] * (rxy[i] - 1) * scaling;
+      }
+      errorState = OnlineCalculatorError.None;
+      // return 30.0 * (q - 2 * (n - 2) * r + (n - 2) * (n - 3) * s) / n / (n - 1) / (n - 2) / (n - 3) / (n - 4);
+      double t0 = q / (n - 2) / (n - 3) / (n - 4);
+      double t1 = 2 * r / (n - 3) / (n - 4);
+      double t2 = s / (n - 4);
+      return 30.0 * (t0 - t1 + t2);
+    }
+
+    private static double[] TiedRank(IEnumerable<double> xs) {
+      var xsArr = xs.ToArray();
+      var idx = Enumerable.Range(1, xsArr.Length).ToArray();
+      Array.Sort(xsArr, idx);
+      CRank(xsArr);
+      Array.Sort(idx, xsArr);
+      return xsArr;
+    }
+
+    /// <summary>
+    /// Calculates the joint rank with midranks for ties. Source: hoeffd.r from R package hmisc http://cran.r-project.org/web/packages/Hmisc/index.html
+    /// </summary>
+    /// <param name="xs"></param>
+    /// <param name="ys"></param>
+    /// <returns></returns>
+    private static double[] TiedRank(IEnumerable<double> xs, IEnumerable<double> ys) {
+      var xsArr = xs.ToArray();
+      var ysArr = ys.ToArray();
+      var r = new double[xsArr.Length];
+      int n = r.Length;
+      for (int i = 0; i < n; i++) {
+        var xi = xsArr[i];
+        var yi = ysArr[i];
+        double ri = 0.0;
+        for (int j = 0; j < n; j++) {
+          if (i != j) {
+            double cx;
+            if (xsArr[j] < xi) cx = 1.0;
+            else if (xsArr[j] > xi) cx = 0.0;
+            else cx = 0.5;  // eq
+            double cy;
+            if (ysArr[j] < yi) cy = 1.0;
+            else if (ysArr[j] > yi) cy = 0.0;
+            else cy = 0.5; // eq
+            ri = ri + cx * cy;
+          }
+        }
+        r[i] = ri;
+      }
+      return r;
+    }
+
+    /// <summary>
+    /// Calculates midranks. Source: Numerical Recipes in C.
+    /// </summary>
+    /// <param name="w">Sorted array of elements, replaces the elements by their rank, including midranking of ties</param>
+    /// <returns></returns>
+    private static void CRank(double[] w) {
+      int i = 0;
+      int n = w.Length;
+      while (i < n - 1) {
+        if (w[i + 1] > w[i]) {    // w[i+1] must be larger or equal w[i] as w must be sorted
+          // not a tie
+          w[i] = i + 1;
+          i++;
+        } else {
+          int j;
+          for (j = i + 1; j < n && w[j] <= w[i]; j++) ; // how far does it go (<= effectively means == as w must be sorted)
+          double rank = 0.5 * (i + j - 1);
+          int k;
+          for (k = i; k < j - 1; k++) w[k] = rank; // set the rank for all tied entries
+          i = j;
+        }
+      }
+
+      if (i == n - 1) w[n - 1] = n - 1;   // if the last element was not tied, this is its rank
+    }
+  }
+}
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Plugin.cs.frame
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Plugin.cs.frame	(revision 7832)
+++ branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/Plugin.cs.frame	(revision 8085)
@@ -36,4 +36,5 @@
   [PluginDependency("HeuristicLab.Parameters", "3.3")]
   [PluginDependency("HeuristicLab.Persistence", "3.3")]
+  [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
   public class HeuristicLabProblemsDataAnalysisPlugin : PluginBase {
   }
Index: branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/TableFileParser.cs
===================================================================
--- branches/GP-MoveOperators/HeuristicLab.Problems.DataAnalysis/3.4/TableFileParser.cs	(revision 7832)
+++ 	(revision )
@@ -1,435 +1,0 @@
-#region License Information
-/* HeuristicLab
- * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
- *
- * This file is part of HeuristicLab.
- *
- * HeuristicLab is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * HeuristicLab is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
- */
-#endregion
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
-using System.Linq;
-using System.Runtime.Serialization;
-using System.Text;
-
-namespace HeuristicLab.Problems.DataAnalysis {
-  public class TableFileParser {
-    private const int BUFFER_SIZE = 1024;
-    private readonly char[] POSSIBLE_SEPARATORS = new char[] { ',', ';', '\t' };
-    private Tokenizer tokenizer;
-    private List<List<object>> rowValues;
-
-    private int rows;
-    public int Rows {
-      get { return rows; }
-      set { rows = value; }
-    }
-
-    private int columns;
-    public int Columns {
-      get { return columns; }
-      set { columns = value; }
-    }
-
-    private List<IList> values;
-    public List<IList> Values {
-      get {
-        return values;
-      }
-    }
-
-    private List<string> variableNames;
-    public IEnumerable<string> VariableNames {
-      get {
-        if (variableNames.Count > 0) return variableNames;
-        else {
-          string[] names = new string[columns];
-          for (int i = 0; i < names.Length; i++) {
-            names[i] = "X" + i.ToString("000");
-          }
-          return names;
-        }
-      }
-    }
-
-    public TableFileParser() {
-      rowValues = new List<List<object>>();
-      variableNames = new List<string>();
-    }
-
-    public void Parse(string fileName) {
-      NumberFormatInfo numberFormat;
-      DateTimeFormatInfo dateTimeFormatInfo;
-      char separator;
-      DetermineFileFormat(fileName, out numberFormat, out dateTimeFormatInfo, out separator);
-      using (StreamReader reader = new StreamReader(fileName)) {
-        tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
-        // parse the file
-        Parse();
-      }
-
-      // translate the list of samples into a DoubleMatrixData item
-      rows = rowValues.Count;
-      columns = rowValues[0].Count;
-      values = new List<IList>();
-
-      //create columns
-      for (int col = 0; col < columns; col++) {
-        var types = rowValues.Select(r => r[col]).Where(v => v != null && v as string != string.Empty).Take(10).Select(v => v.GetType());
-        if (!types.Any()) {
-          values.Add(new List<string>());
-          continue;
-        }
-
-        var columnType = types.GroupBy(v => v).OrderBy(v => v.Count()).Last().Key;
-        if (columnType == typeof(double)) values.Add(new List<double>());
-        else if (columnType == typeof(DateTime)) values.Add(new List<DateTime>());
-        else if (columnType == typeof(string)) values.Add(new List<string>());
-        else throw new InvalidOperationException();
-      }
-
-
-
-      //fill with values
-      foreach (List<object> row in rowValues) {
-        int columnIndex = 0;
-        foreach (object element in row) {
-          if (values[columnIndex] is List<double> && !(element is double))
-            values[columnIndex].Add(double.NaN);
-          else if (values[columnIndex] is List<DateTime> && !(element is DateTime))
-            values[columnIndex].Add(DateTime.MinValue);
-          else if (values[columnIndex] is List<string> && !(element is string))
-            values[columnIndex].Add(string.Empty);
-          else
-            values[columnIndex].Add(element);
-          columnIndex++;
-        }
-      }
-    }
-
-    private void DetermineFileFormat(string fileName, out NumberFormatInfo numberFormat, out DateTimeFormatInfo dateTimeFormatInfo, out char separator) {
-      using (StreamReader reader = new StreamReader(fileName)) {
-        // skip first line
-        reader.ReadLine();
-        // read a block
-        char[] buffer = new char[BUFFER_SIZE];
-        int charsRead = reader.ReadBlock(buffer, 0, BUFFER_SIZE);
-        // count frequency of special characters
-        Dictionary<char, int> charCounts = buffer.Take(charsRead)
-          .GroupBy(c => c)
-          .ToDictionary(g => g.Key, g => g.Count());
-
-        // depending on the characters occuring in the block 
-        // we distinghish a number of different cases based on the the following rules:
-        // many points => it must be English number format, the other frequently occuring char is the separator
-        // no points but many commas => this is the problematic case. Either German format (real numbers) or English format (only integer numbers) with ',' as separator
-        //   => check the line in more detail:
-        //            English: 0, 0, 0, 0
-        //            German:  0,0 0,0 0,0 ...
-        //            => if commas are followed by space => English format
-        // no points no commas => English format (only integer numbers) use the other frequently occuring char as separator
-        // in all cases only treat ' ' as separator if no other separator is possible (spaces can also occur additionally to separators)
-        if (OccurrencesOf(charCounts, '.') > 10) {
-          numberFormat = NumberFormatInfo.InvariantInfo;
-          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-          separator = POSSIBLE_SEPARATORS
-            .Where(c => OccurrencesOf(charCounts, c) > 10)
-            .OrderBy(c => -OccurrencesOf(charCounts, c))
-            .DefaultIfEmpty(' ')
-            .First();
-        } else if (OccurrencesOf(charCounts, ',') > 10) {
-          // no points and many commas
-          // count the number of tokens (chains of only digits and commas) that contain multiple comma characters
-          int tokensWithMultipleCommas = 0;
-          for (int i = 0; i < charsRead; i++) {
-            int nCommas = 0;
-            while (i < charsRead && (buffer[i] == ',' || Char.IsDigit(buffer[i]))) {
-              if (buffer[i] == ',') nCommas++;
-              i++;
-            }
-            if (nCommas > 2) tokensWithMultipleCommas++;
-          }
-          if (tokensWithMultipleCommas > 1) {
-            // English format (only integer values) with ',' as separator
-            numberFormat = NumberFormatInfo.InvariantInfo;
-            dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-            separator = ',';
-          } else {
-            char[] disallowedSeparators = new char[] { ',' };
-            // German format (real values)
-            numberFormat = NumberFormatInfo.GetInstance(new CultureInfo("de-DE"));
-            dateTimeFormatInfo = DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE"));
-            separator = POSSIBLE_SEPARATORS
-              .Except(disallowedSeparators)
-              .Where(c => OccurrencesOf(charCounts, c) > 10)
-              .OrderBy(c => -OccurrencesOf(charCounts, c))
-              .DefaultIfEmpty(' ')
-              .First();
-          }
-        } else {
-          // no points and no commas => English format
-          numberFormat = NumberFormatInfo.InvariantInfo;
-          dateTimeFormatInfo = DateTimeFormatInfo.InvariantInfo;
-          separator = POSSIBLE_SEPARATORS
-            .Where(c => OccurrencesOf(charCounts, c) > 10)
-            .OrderBy(c => -OccurrencesOf(charCounts, c))
-            .DefaultIfEmpty(' ')
-            .First();
-        }
-      }
-    }
-
-    private int OccurrencesOf(Dictionary<char, int> charCounts, char c) {
-      return charCounts.ContainsKey(c) ? charCounts[c] : 0;
-    }
-
-    #region tokenizer
-    internal enum TokenTypeEnum {
-      NewLine, Separator, String, Double, DateTime
-    }
-
-    internal class Token {
-      public TokenTypeEnum type;
-      public string stringValue;
-      public double doubleValue;
-      public DateTime dateTimeValue;
-
-      public Token(TokenTypeEnum type, string value) {
-        this.type = type;
-        stringValue = value;
-        dateTimeValue = DateTime.MinValue;
-        doubleValue = 0.0;
-      }
-
-      public override string ToString() {
-        return stringValue;
-      }
-    }
-
-
-    internal class Tokenizer {
-      private StreamReader reader;
-      private List<Token> tokens;
-      private NumberFormatInfo numberFormatInfo;
-      private DateTimeFormatInfo dateTimeFormatInfo;
-      private char separator;
-      private const string INTERNAL_SEPARATOR = "#";
-
-      private int currentLineNumber = 0;
-      public int CurrentLineNumber {
-        get { return currentLineNumber; }
-        private set { currentLineNumber = value; }
-      }
-      private string currentLine;
-      public string CurrentLine {
-        get { return currentLine; }
-        private set { currentLine = value; }
-      }
-
-      private Token newlineToken;
-      public Token NewlineToken {
-        get { return newlineToken; }
-        private set { newlineToken = value; }
-      }
-      private Token separatorToken;
-      public Token SeparatorToken {
-        get { return separatorToken; }
-        private set { separatorToken = value; }
-      }
-
-      public Tokenizer(StreamReader reader, NumberFormatInfo numberFormatInfo, DateTimeFormatInfo dateTimeFormatInfo, char separator) {
-        this.reader = reader;
-        this.numberFormatInfo = numberFormatInfo;
-        this.dateTimeFormatInfo = dateTimeFormatInfo;
-        this.separator = separator;
-        separatorToken = new Token(TokenTypeEnum.Separator, INTERNAL_SEPARATOR);
-        newlineToken = new Token(TokenTypeEnum.NewLine, Environment.NewLine);
-        tokens = new List<Token>();
-        ReadNextTokens();
-      }
-
-      private void ReadNextTokens() {
-        if (!reader.EndOfStream) {
-          CurrentLine = reader.ReadLine();
-          var newTokens = from str in Split(CurrentLine)
-                          let trimmedStr = str.Trim()
-                          where !string.IsNullOrEmpty(trimmedStr)
-                          select MakeToken(trimmedStr);
-
-          tokens.AddRange(newTokens);
-          tokens.Add(NewlineToken);
-          CurrentLineNumber++;
-        }
-      }
-
-      private IEnumerable<string> Split(string line) {
-        StringBuilder subStr = new StringBuilder();
-        foreach (char c in line) {
-          if (c == separator) {
-            yield return subStr.ToString();
-            subStr = new StringBuilder();
-            // all separator characters are transformed to the internally used separator character
-            yield return INTERNAL_SEPARATOR;
-          } else {
-            subStr.Append(c);
-          }
-        }
-        yield return subStr.ToString();
-      }
-
-      private Token MakeToken(string strToken) {
-        Token token = new Token(TokenTypeEnum.String, strToken);
-        if (strToken.Equals(INTERNAL_SEPARATOR)) {
-          return SeparatorToken;
-        } else if (double.TryParse(strToken, NumberStyles.Float, numberFormatInfo, out token.doubleValue)) {
-          token.type = TokenTypeEnum.Double;
-          return token;
-        } else if (DateTime.TryParse(strToken, dateTimeFormatInfo, DateTimeStyles.None, out token.dateTimeValue)) {
-          token.type = TokenTypeEnum.DateTime;
-          return token;
-        }
-
-        // couldn't parse the token as an int or float number  or datetime value so return a string token
-        return token;
-      }
-
-      public Token Peek() {
-        return tokens[0];
-      }
-
-      public Token Next() {
-        Token next = tokens[0];
-        tokens.RemoveAt(0);
-        if (tokens.Count == 0) {
-          ReadNextTokens();
-        }
-        return next;
-      }
-
-      public bool HasNext() {
-        return tokens.Count > 0 || !reader.EndOfStream;
-      }
-    }
-    #endregion
-
-    #region parsing
-    private void Parse() {
-      ParseVariableNames();
-      if (!tokenizer.HasNext()) Error("Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
-      ParseValues();
-      if (rowValues.Count == 0) Error("Couldn't parse data values. Probably because of incorrect number format (the parser expects english number format with a '.' as decimal separator).", "", tokenizer.CurrentLineNumber);
-    }
-
-    private void ParseValues() {
-      while (tokenizer.HasNext()) {
-        if (tokenizer.Peek() == tokenizer.NewlineToken) {
-          tokenizer.Next();
-        } else {
-          List<object> row = new List<object>();
-          object value = NextValue(tokenizer);
-          row.Add(value);
-          while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
-            Expect(tokenizer.SeparatorToken);
-            row.Add(NextValue(tokenizer));
-          }
-          Expect(tokenizer.NewlineToken);
-          // all rows have to have the same number of values            
-          // the first row defines how many samples are needed
-          if (rowValues.Count > 0 && rowValues[0].Count != row.Count) {
-            Error("The first row of the dataset has " + rowValues[0].Count + " columns." +
-                  "\nLine " + tokenizer.CurrentLineNumber + " has " + row.Count + " columns.", "",
-                  tokenizer.CurrentLineNumber);
-          }
-          rowValues.Add(row);
-        }
-      }
-    }
-
-    private object NextValue(Tokenizer tokenizer) {
-      if (tokenizer.Peek() == tokenizer.SeparatorToken || tokenizer.Peek() == tokenizer.NewlineToken) return string.Empty;
-      Token current = tokenizer.Next();
-      if (current.type == TokenTypeEnum.Separator) {
-        return double.NaN;
-      } else if (current.type == TokenTypeEnum.String) {
-        return current.stringValue;
-      } else if (current.type == TokenTypeEnum.Double) {
-        return current.doubleValue;
-      } else if (current.type == TokenTypeEnum.DateTime) {
-        return current.dateTimeValue;
-      }
-      // found an unexpected token => throw error 
-      Error("Unexpected token.", current.stringValue, tokenizer.CurrentLineNumber);
-      // this line is never executed because Error() throws an exception
-      throw new InvalidOperationException();
-    }
-
-    private void ParseVariableNames() {
-      //if first token is double no variables names are given
-      if (tokenizer.Peek().type == TokenTypeEnum.Double) return;
-
-      // the first line must contain variable names
-      List<Token> tokens = new List<Token>();
-      Token valueToken;
-      valueToken = tokenizer.Next();
-      tokens.Add(valueToken);
-      while (tokenizer.HasNext() && tokenizer.Peek() == tokenizer.SeparatorToken) {
-        Expect(tokenizer.SeparatorToken);
-        valueToken = tokenizer.Next();
-        if (valueToken != tokenizer.NewlineToken) {
-          tokens.Add(valueToken);
-        }
-      }
-      if (valueToken != tokenizer.NewlineToken) {
-        Expect(tokenizer.NewlineToken);
-      }
-      variableNames = tokens.Select(x => x.stringValue.Trim()).ToList();
-    }
-
-    private void Expect(Token expectedToken) {
-      Token actualToken = tokenizer.Next();
-      if (actualToken != expectedToken) {
-        Error("Expected: " + expectedToken, actualToken.stringValue, tokenizer.CurrentLineNumber);
-      }
-    }
-
-    private void Error(string message, string token, int lineNumber) {
-      throw new DataFormatException("Error while parsing.\n" + message, token, lineNumber);
-    }
-    #endregion
-
-    [Serializable]
-    private class DataFormatException : Exception {
-      private int line;
-      public int Line {
-        get { return line; }
-      }
-      private string token;
-      public string Token {
-        get { return token; }
-      }
-      public DataFormatException(string message, string token, int line)
-        : base(message + "\nToken: " + token + " (line: " + line + ")") {
-        this.token = token;
-        this.line = line;
-      }
-
-      public DataFormatException(SerializationInfo info, StreamingContext context) : base(info, context) { }
-    }
-  }
-}
