Changeset 14240 for branches/symbreg-factors-2650

Timestamp:

08/05/16 18:44:51 (8 years ago)

Author:

gkronber

Message:

#2650: added support for categorical variables to LDA and MNL (TODO: OneR )

Location:

branches/symbreg-factors-2650

Files:

• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs (modified) (2 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/FactorVariable.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionComparisonView.cs (modified) (1 diff)

branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/AlglibUtil.cs

@@ -118 +118 @@
       return matrix;
     }
+
+    public static IEnumerable<KeyValuePair<string, IEnumerable<string>>> GetFactorVariableValues(IDataset ds, IEnumerable<string> factorVariables, IEnumerable<int> rows) {
+      return from factor in factorVariables
+             let distinctValues = ds.GetStringValues(factor, rows).Distinct().ToArray()
+             // 1 distinct value => skip (constant)
+             // 2 distinct values => only take one of the two values
+             // >=3 distinct values => create a binary value for each value
+             let reducedValues = distinctValues.Length <= 2
+               ? distinctValues.Take(distinctValues.Length - 1)
+               : distinctValues
+             select new KeyValuePair<string, IEnumerable<string>>(factor, reducedValues);
+    }
   }
 }

branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearDiscriminantAnalysis.cs

@@ -36 +36 @@
   /// Linear discriminant analysis classification algorithm.
   /// </summary>
-  [Item("Linear Discriminant Analysis", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
+  [Item("Linear Discriminant Analysis (LDA)", "Linear discriminant analysis classification algorithm (wrapper for ALGLIB).")]
   [Creatable(CreatableAttribute.Categories.DataAnalysisClassification, Priority = 100)]
   [StorableClass]
@@ -70 +70 @@
       IEnumerable<int> rows = problemData.TrainingIndices;
       int nClasses = problemData.ClassNames.Count();
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      var doubleVariableNames = allowedInputVariables.Where(dataset.VariableHasType<double>).ToArray();
+      var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>).ToArray();
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariables = AlglibUtil.GetFactorVariableValues(dataset, factorVariableNames, rows);
+      double[,] factorMatrix = AlglibUtil.PrepareInputMatrix(dataset, factorVariables, rows);
+
+      inputMatrix = factorMatrix.VertCat(inputMatrix);
+
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Linear discriminant analysis does not support NaN or infinity values in the input dataset.");
@@ -82 +90 @@
       int info;
       double[] w;
-      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), allowedInputVariables.Count(), nClasses, out info, out w);
+      alglib.fisherlda(inputMatrix, inputMatrix.GetLength(0), inputMatrix.GetLength(1) - 1, nClasses, out info, out w);
       if (info < 1) throw new ArgumentException("Error in calculation of linear discriminant analysis solution");
 
@@ -92 +100 @@
 
       int col = 0;
-      foreach (string column in allowedInputVariables) {
+      foreach (var kvp in factorVariables) {
+        var varName = kvp.Key;
+        foreach (var cat in kvp.Value) {
+          FactorVariableTreeNode vNode =
+            (FactorVariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.FactorVariable().CreateTreeNode();
+          vNode.VariableName = varName;
+          vNode.VariableValue = cat;
+          vNode.Weight = w[col];
+          addition.AddSubtree(vNode);
+          col++;
+        }
+      }
+      foreach (string column in doubleVariableNames) {
         VariableTreeNode vNode = (VariableTreeNode)new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
         vNode.VariableName = column;

branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/LinearRegression.cs

@@ -75 +75 @@
       var doubleVariables = allowedInputVariables.Where(dataset.VariableHasType<double>);
       var factorVariableNames = allowedInputVariables.Where(dataset.VariableHasType<string>);
-      var factorVariables = from factor in factorVariableNames
-                            let distinctValues = dataset.GetStringValues(factor, rows).Distinct().ToArray()
-                            // 1 distinct value => skip (constant)
-                            // 2 distinct values => only take one of the two values
-                            // >=3 distinct values => create a binary value for each value
-                            let reducedValues = distinctValues.Length <= 2
-                              ? distinctValues.Take(distinctValues.Length - 1)
-                              : distinctValues
-                            select new KeyValuePair<string, IEnumerable<string>>(factor, reducedValues);
+      var factorVariables = AlglibUtil.GetFactorVariableValues(dataset, factorVariableNames, rows);
       double[,] binaryMatrix = AlglibUtil.PrepareInputMatrix(dataset, factorVariables, rows);
       double[,] doubleVarMatrix = AlglibUtil.PrepareInputMatrix(dataset, doubleVariables.Concat(new string[] { targetVariable }), rows);

branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitClassification.cs

@@ -68 +68 @@
       var dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
-      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      var doubleVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<double>);
+      var factorVariableNames = problemData.AllowedInputVariables.Where(dataset.VariableHasType<string>);
       IEnumerable<int> rows = problemData.TrainingIndices;
-      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, doubleVariableNames.Concat(new string[] { targetVariable }), rows);
+
+      var factorVariableValues = AlglibUtil.GetFactorVariableValues(dataset, factorVariableNames, rows);
+      var factorMatrix = AlglibUtil.PrepareInputMatrix(dataset, factorVariableValues, rows);
+      inputMatrix = factorMatrix.VertCat(inputMatrix);
+
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
         throw new NotSupportedException("Multinomial logit classification does not support NaN or infinity values in the input dataset.");
@@ -95 +101 @@
       relClassError = alglib.mnlrelclserror(lm, inputMatrix, nRows);
 
-      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, allowedInputVariables, classValues), (IClassificationProblemData)problemData.Clone());
+      MultinomialLogitClassificationSolution solution = new MultinomialLogitClassificationSolution(new MultinomialLogitModel(lm, targetVariable, doubleVariableNames, factorVariableValues, classValues), (IClassificationProblemData)problemData.Clone());
       return solution;
     }

branches/symbreg-factors-2650/HeuristicLab.Algorithms.DataAnalysis/3.4/Linear/MultinomialLogitModel.cs

@@ -56 +56 @@
     [Storable]
     private double[] classValues;
+    [Storable]
+    private List<KeyValuePair<string, IEnumerable<string>>> factorVariables;
+
     [StorableConstructor]
     private MultinomialLogitModel(bool deserializing)
@@ -68 +71 @@
       allowedInputVariables = (string[])original.allowedInputVariables.Clone();
       classValues = (double[])original.classValues.Clone();
+      this.factorVariables = original.factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
     }
-    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues)
+    public MultinomialLogitModel(alglib.logitmodel logitModel, string targetVariable, IEnumerable<string> doubleInputVariables, IEnumerable<KeyValuePair<string, IEnumerable<string>>> factorVariables, double[] classValues)
       : base(targetVariable) {
       this.name = ItemName;
       this.description = ItemDescription;
       this.logitModel = logitModel;
-      this.allowedInputVariables = allowedInputVariables.ToArray();
+      this.allowedInputVariables = doubleInputVariables.ToArray();
+      this.factorVariables = factorVariables.Select(kvp => new KeyValuePair<string, IEnumerable<string>>(kvp.Key, new List<string>(kvp.Value))).ToList();
       this.classValues = (double[])classValues.Clone();
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      factorVariables = new List<KeyValuePair<string, IEnumerable<string>>>();
+      #endregion
     }
 
@@ -83 +96 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IDataset dataset, IEnumerable<int> rows) {
+
       double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+      double[,] factorData = AlglibUtil.PrepareInputMatrix(dataset, factorVariables, rows);
+
+      inputData = factorData.VertCat(inputData);
 
       int n = inputData.GetLength(0);

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Symbols/FactorVariable.cs

@@ -28 +28 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
-  // TODO: handle correctly in all interpreters and formatters
-  // TODO: view for factor variables (configuration of actually allowed factors)
-  // TODO: handle correctly in variable impacts view
-  // TODO: handle correctly in Non-linear regression (infix parser and infix formatter)
-  // TODO: support in all analyzers which handle variable symbols specifically
-  // TODO: handle correctly in gradient views
-  // TODO: handle correctly in ERC view (create linear regression model)
-  // TODO: handle correctly in classification - solution comparison
-  // TODO: allow factor variables in decision trees (and therefore GBT)?
-  // TODO: support in more algs?
-  // TODO: support in more views?
   [StorableClass]
   [Item("FactorVariable", "Represents a categorical variable (comparable to factors as in R).")]

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionComparisonView.cs

@@ -131 +131 @@
       zeroR.Name = "ZeroR Classification Solution";
       newSolutions.Add(zeroR);
-      var oneR = OneR.CreateOneRSolution(problemData);
-      oneR.Name = "OneR Classification Solution";
-      newSolutions.Add(oneR);
+      try {
+        var oneR = OneR.CreateOneRSolution(problemData);
+        oneR.Name = "OneR Classification Solution";
+        newSolutions.Add(oneR);
+      } catch (NotSupportedException) { } catch (ArgumentException) { }
       try {
         var lda = LinearDiscriminantAnalysis.CreateLinearDiscriminantAnalysisSolution(problemData);