Changeset 11337 for trunk/sources/HeuristicLab.Algorithms.DataAnalysis

Timestamp:

09/03/14 15:12:04 (10 years ago)

Author:

bburlacu

Message:

#2234: Refactored SVM grid search, added support for symbolic classification.

Location:

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine

Files:

• SupportVectorClassification.cs (modified) (3 diffs)
• SupportVectorMachineUtil.cs (modified) (5 diffs)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorClassification.cs

@@ -142 +142 @@
 
     public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
-      string svmType, string kernelType, double cost, double nu, double gamma, int degree,
-      out double trainingAccuracy, out double testAccuracy, out int nSv) {
+      string svmType, string kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
+      return CreateSupportVectorClassificationSolution(problemData, allowedInputVariables, GetKernelType(svmType), GetKernelType(kernelType), cost, nu, gamma, degree,
+        out trainingAccuracy, out testAccuracy, out nSv);
+    }
+
+    public static SupportVectorClassificationSolution CreateSupportVectorClassificationSolution(IClassificationProblemData problemData, IEnumerable<string> allowedInputVariables,
+      int svmType, int kernelType, double cost, double nu, double gamma, int degree, out double trainingAccuracy, out double testAccuracy, out int nSv) {
       Dataset dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
@@ -150 +155 @@
       //extract SVM parameters from scope and set them
       svm_parameter parameter = new svm_parameter();
-      parameter.svm_type = GetSvmType(svmType);
-      parameter.kernel_type = GetKernelType(kernelType);
+      parameter.svm_type = svmType;
+      parameter.kernel_type = kernelType;
       parameter.C = cost;
       parameter.nu = nu;
@@ -161 +166 @@
       parameter.shrinking = 1;
       parameter.coef0 = 0;
-
 
       var weightLabels = new List<int>();

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/SupportVectorMachine/SupportVectorMachineUtil.cs

@@ -39 +39 @@
     /// <returns>A problem data type that can be used to train a support vector machine.</returns>
     public static svm_problem CreateSvmProblem(Dataset dataset, string targetVariable, IEnumerable<string> inputVariables, IEnumerable<int> rowIndices) {
-      double[] targetVector =
-        dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
-
+      double[] targetVector = dataset.GetDoubleValues(targetVariable, rowIndices).ToArray();
       svm_node[][] nodes = new svm_node[targetVector.Length][];
-      List<svm_node> tempRow;
       int maxNodeIndex = 0;
       int svmProblemRowIndex = 0;
       List<string> inputVariablesList = inputVariables.ToList();
       foreach (int row in rowIndices) {
-        tempRow = new List<svm_node>();
+        List<svm_node> tempRow = new List<svm_node>();
         int colIndex = 1; // make sure the smallest node index for SVM = 1
         foreach (var inputVariable in inputVariablesList) {
@@ -62 +59 @@
         nodes[svmProblemRowIndex++] = tempRow.ToArray();
       }
-
-      return new svm_problem() { l = targetVector.Length, y = targetVector, x = nodes };
+      return new svm_problem { l = targetVector.Length, y = targetVector, x = nodes };
     }
 
@@ -89 +85 @@
 
     /// <summary>
-    /// Generate a collection of training indices corresponding to folds in the data (used for crossvalidation)
+    /// Generate a collection of row sequences corresponding to folds in the data (used for crossvalidation)
     /// </summary>
     /// <remarks>This method is aimed to be lightweight and as such does not clone the dataset.</remarks>
     /// <param name="problemData">The problem data</param>
-    /// <param name="nFolds">The number of folds to generate</param>
+    /// <param name="numberOfFolds">The number of folds to generate</param>
     /// <returns>A sequence of folds representing each a sequence of row numbers</returns>
-    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int nFolds) {
+    public static IEnumerable<IEnumerable<int>> GenerateFolds(IDataAnalysisProblemData problemData, int numberOfFolds) {
       int size = problemData.TrainingPartition.Size;
-
-      int foldSize = size / nFolds; // rounding to integer
-      var trainingIndices = problemData.TrainingIndices;
-
-      for (int i = 0; i < nFolds; ++i) {
-        int n = i * foldSize;
-        int s = n + 2 * foldSize > size ? foldSize + size % foldSize : foldSize;
-        yield return trainingIndices.Skip(n).Take(s);
+      int f = size / numberOfFolds, r = size % numberOfFolds; // number of folds rounded to integer and remainder
+      int start = 0, end = f;
+      for (int i = 0; i < numberOfFolds; ++i) {
+        if (r > 0) { ++end; --r; }
+        yield return problemData.TrainingIndices.Skip(start).Take(end - start);
+        start = end;
+        end += f;
       }
     }
 
-    public static void CrossValidate(IDataAnalysisProblemData problemData, svm_parameter parameters, int numFolds, out double avgTestMse) {
-      avgTestMse = 0;
-      var folds = GenerateFolds(problemData, numFolds).ToList();
-      var calc = new OnlineMeanSquaredErrorCalculator();
+    private static Tuple<svm_problem, svm_problem>[] GenerateSvmPartitions(IDataAnalysisProblemData problemData, int numberOfFolds) {
+      var folds = GenerateFolds(problemData, numberOfFolds).ToList();
       var targetVariable = GetTargetVariableName(problemData);
-      for (int i = 0; i < numFolds; ++i) {
+      var partitions = new Tuple<svm_problem, svm_problem>[numberOfFolds];
+      for (int i = 0; i < numberOfFolds; ++i) {
         int p = i; // avoid "access to modified closure" warning below
-        var training = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
+        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
         var testRows = folds[i];
-        var trainingSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, training);
+        var trainingSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, trainingRows);
         var testSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, testRows);
+        partitions[i] = new Tuple<svm_problem, svm_problem>(trainingSvmProblem, testSvmProblem);
+      }
+      return partitions;
+    }
 
-        var model = svm.svm_train(trainingSvmProblem, parameters);
-        calc.Reset();
-        for (int j = 0; j < testSvmProblem.l; ++j)
-          calc.Add(testSvmProblem.y[j], svm.svm_predict(model, testSvmProblem.x[j]));
-        avgTestMse += calc.MeanSquaredError;
-      }
-      avgTestMse /= numFolds;
+    public static void CrossValidate(IDataAnalysisProblemData problemData, svm_parameter parameters, int numberOfFolds, out double avgTestMse) {
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds);
+      CrossValidate(problemData, parameters, partitions, out avgTestMse);
     }
 
@@ -156 +150 @@
       DoubleValue mse = new DoubleValue(Double.MaxValue);
       var bestParam = DefaultParameters();
-
-      // search for C, gamma and epsilon parameter combinations
       var pNames = parameterRanges.Keys.ToList();
       var pRanges = pNames.Select(x => parameterRanges[x]);
-
       var crossProduct = pRanges.CartesianProduct();
       var setters = pNames.Select(GenerateSetter).ToList();
-      var folds = GenerateFolds(problemData, numberOfFolds).ToList();
-
-      var partitions = new Tuple<svm_problem, svm_problem>[numberOfFolds];
-      var targetVariable = GetTargetVariableName(problemData);
-
-      for (int i = 0; i < numberOfFolds; ++i) {
-        int p = i; // avoid "access to modified closure" warning below
-        var trainingRows = folds.SelectMany((par, j) => j != p ? par : Enumerable.Empty<int>());
-        var testRows = folds[i];
-        var trainingSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, trainingRows);
-        var testSvmProblem = CreateSvmProblem(problemData.Dataset, targetVariable, problemData.AllowedInputVariables, testRows);
-        partitions[i] = new Tuple<svm_problem, svm_problem>(trainingSvmProblem, testSvmProblem);
-      }
-
+      var partitions = GenerateSvmPartitions(problemData, numberOfFolds);
       Parallel.ForEach(crossProduct, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, nuple => {
-        //  foreach (var nuple in crossProduct) {
         var list = nuple.ToList();
         var parameters = DefaultParameters();
@@ -189 +166 @@
         if (testMse < mse.Value) {
           lock (mse) { mse.Value = testMse; }
-          lock (bestParam) { bestParam = (svm_parameter)parameters.Clone(); } // set best parameter values to the best found so far 
+          lock (bestParam) { bestParam = (svm_parameter)parameters.Clone(); }
         }
       });