Changeset 2367 for trunk/sources/HeuristicLab.LinearRegression

Timestamp:

09/17/09 10:15:56 (15 years ago)

Author:

gkronber

Message:

Fixed #750 (LinearRegressionOperator should first remove columns with many NaN values and after that remove the rows with NaN values).

File:

• trunk/sources/HeuristicLab.LinearRegression/3.2/LinearRegressionOperator.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.LinearRegression/3.2/LinearRegressionOperator.cs

@@ -56 +56 @@
       int minTimeOffset = minTimeOffsetData == null ? 0 : minTimeOffsetData.Data;
 
-      List<int> allowedRows = CalculateAllowedRows(dataset, targetVariable, start, end, minTimeOffset, maxTimeOffset);
       List<int> allowedColumns = CalculateAllowedColumns(dataset, targetVariable, start, end);
+      List<int> allowedRows = CalculateAllowedRows(dataset, targetVariable, allowedColumns, start, end, minTimeOffset, maxTimeOffset);
 
       double[,] inputMatrix = PrepareInputMatrix(dataset, allowedColumns, allowedRows, minTimeOffset, maxTimeOffset);
@@ -110 +110 @@
 
     //returns list of valid row indexes (rows without NaN values)
-    private List<int> CalculateAllowedRows(Dataset dataset, int targetVariable, int start, int end, int minTimeOffset, int maxTimeOffset) {
+    private List<int> CalculateAllowedRows(Dataset dataset, int targetVariable, IList<int> allowedColumns, int start, int end, int minTimeOffset, int maxTimeOffset) {
       List<int> allowedRows = new List<int>();
       bool add;
       for (int row = start; row < end; row++) {
         add = true;
-        for (int col = 0; col < dataset.Columns && add == true; col++) {
+        for (int colIndex = 0; colIndex < allowedColumns.Count && add == true; colIndex++) {
           for (int timeOffset = minTimeOffset; timeOffset <= maxTimeOffset; timeOffset++) {
             if (
               row + timeOffset < 0 ||
               row + timeOffset > dataset.Rows ||
-              double.IsNaN(dataset.GetValue(row + timeOffset, col)) ||
+              double.IsNaN(dataset.GetValue(row + timeOffset, allowedColumns[colIndex])) ||
+              double.IsInfinity(dataset.GetValue(row + timeOffset, allowedColumns[colIndex])) ||
               double.IsNaN(dataset.GetValue(row + timeOffset, targetVariable))) {
               add = false;
@@ -133 +134 @@
     }
 
-    //returns list of valid column indexes (columns which contain at least one non-zero value)
+    //returns list of valid column indexes (columns which contain max. 10% NaN (or infinity) and contain at least two different values)
     private List<int> CalculateAllowedColumns(Dataset dataset, int targetVariable, int start, int end) {
       List<int> allowedColumns = new List<int>();
+      double n = end - start;
       for (int i = 0; i < dataset.Columns; i++) {
-        if (i == targetVariable) continue;
-        if (!dataset.GetMinimum(i, start, end).IsAlmost(0.0) ||
-            !dataset.GetMaximum(i, start, end).IsAlmost(0.0))
+        double nanRatio = CountNaN(dataset, i, start, end) / n;
+        if (i != targetVariable && nanRatio < 0.1 && dataset.GetRange(i, start, end) > 0.0) {
           allowedColumns.Add(i);
+        }
       }
       return allowedColumns;
     }
+
+    private double CountNaN(Dataset dataset, int column, int start, int end) {
+      double n = 0;
+      for (int i = start; i < end; i++) {
+        if (double.IsNaN(dataset.GetValue(i, column)) || double.IsInfinity(dataset.GetValue(i, column)))
+          n++;
+      }
+      return n;
+    }
+
 
     private double[,] PrepareInputMatrix(Dataset dataset, List<int> allowedColumns, List<int> allowedRows, int minTimeOffset, int maxTimeOffset) {