Changeset 2367

Timestamp:

09/17/09 10:15:56 (15 years ago)

Author:

gkronber

Message:

Fixed #750 (LinearRegressionOperator should first remove columns with many NaN values and after that remove the rows with NaN values).

Location:

trunk/sources

Files:

• HeuristicLab.Common/3.2/HeuristicLab.Common-3.2.csproj (modified) (1 diff)
• HeuristicLab.DataAnalysis/3.2/Dataset.cs (modified) (3 diffs)
• HeuristicLab.LinearRegression/3.2/LinearRegressionOperator.cs (modified) (3 diffs)
• HeuristicLab.sln (modified) (2 diffs)

trunk/sources/HeuristicLab.Common/3.2/HeuristicLab.Common-3.2.csproj

@@ -47 +47 @@
     <DebugType>pdbonly</DebugType>
     <PlatformTarget>x86</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>bin\x64\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
+    <ErrorReport>prompt</ErrorReport>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
+    <OutputPath>bin\x64\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x64</PlatformTarget>
     <ErrorReport>prompt</ErrorReport>
   </PropertyGroup>

trunk/sources/HeuristicLab.DataAnalysis/3.2/Dataset.cs

@@ -34 +34 @@
     private Dictionary<int, Dictionary<int, double>>[] cachedRanges;
     private bool cachedValuesInvalidated = true;
-    
+
     public Dataset()
       : this(new double[,] { { 0.0 } }) {
@@ -128 +128 @@
 
     #region Modify and get values
-    public double GetValue(int i, int j) {
-      return samples[columns * i + j];
+    public double GetValue(int row, int column) {
+      return samples[columns * row + column];
     }
 
@@ -259 +259 @@
         }
         double range = Statistics.Range(values);
-        if (!cachedRanges[column].ContainsKey(start)) cachedRanges[column][start]= new Dictionary<int, double>();
+        if (!cachedRanges[column].ContainsKey(start)) cachedRanges[column][start] = new Dictionary<int, double>();
         cachedRanges[column][start][end] = range;
         return range;

trunk/sources/HeuristicLab.LinearRegression/3.2/LinearRegressionOperator.cs

@@ -56 +56 @@
       int minTimeOffset = minTimeOffsetData == null ? 0 : minTimeOffsetData.Data;
 
-      List<int> allowedRows = CalculateAllowedRows(dataset, targetVariable, start, end, minTimeOffset, maxTimeOffset);
       List<int> allowedColumns = CalculateAllowedColumns(dataset, targetVariable, start, end);
+      List<int> allowedRows = CalculateAllowedRows(dataset, targetVariable, allowedColumns, start, end, minTimeOffset, maxTimeOffset);
 
       double[,] inputMatrix = PrepareInputMatrix(dataset, allowedColumns, allowedRows, minTimeOffset, maxTimeOffset);
@@ -110 +110 @@
 
     //returns list of valid row indexes (rows without NaN values)
-    private List<int> CalculateAllowedRows(Dataset dataset, int targetVariable, int start, int end, int minTimeOffset, int maxTimeOffset) {
+    private List<int> CalculateAllowedRows(Dataset dataset, int targetVariable, IList<int> allowedColumns, int start, int end, int minTimeOffset, int maxTimeOffset) {
       List<int> allowedRows = new List<int>();
       bool add;
       for (int row = start; row < end; row++) {
         add = true;
-        for (int col = 0; col < dataset.Columns && add == true; col++) {
+        for (int colIndex = 0; colIndex < allowedColumns.Count && add == true; colIndex++) {
           for (int timeOffset = minTimeOffset; timeOffset <= maxTimeOffset; timeOffset++) {
             if (
               row + timeOffset < 0 ||
               row + timeOffset > dataset.Rows ||
-              double.IsNaN(dataset.GetValue(row + timeOffset, col)) ||
+              double.IsNaN(dataset.GetValue(row + timeOffset, allowedColumns[colIndex])) ||
+              double.IsInfinity(dataset.GetValue(row + timeOffset, allowedColumns[colIndex])) ||
               double.IsNaN(dataset.GetValue(row + timeOffset, targetVariable))) {
               add = false;
@@ -133 +134 @@
     }
 
-    //returns list of valid column indexes (columns which contain at least one non-zero value)
+    //returns list of valid column indexes (columns which contain max. 10% NaN (or infinity) and contain at least two different values)
     private List<int> CalculateAllowedColumns(Dataset dataset, int targetVariable, int start, int end) {
       List<int> allowedColumns = new List<int>();
+      double n = end - start;
       for (int i = 0; i < dataset.Columns; i++) {
-        if (i == targetVariable) continue;
-        if (!dataset.GetMinimum(i, start, end).IsAlmost(0.0) ||
-            !dataset.GetMaximum(i, start, end).IsAlmost(0.0))
+        double nanRatio = CountNaN(dataset, i, start, end) / n;
+        if (i != targetVariable && nanRatio < 0.1 && dataset.GetRange(i, start, end) > 0.0) {
           allowedColumns.Add(i);
+        }
       }
       return allowedColumns;
     }
+
+    private double CountNaN(Dataset dataset, int column, int start, int end) {
+      double n = 0;
+      for (int i = start; i < end; i++) {
+        if (double.IsNaN(dataset.GetValue(i, column)) || double.IsInfinity(dataset.GetValue(i, column)))
+          n++;
+      }
+      return n;
+    }
+
 
     private double[,] PrepareInputMatrix(Dataset dataset, List<int> allowedColumns, List<int> allowedRows, int minTimeOffset, int maxTimeOffset) {

trunk/sources/HeuristicLab.sln

@@ -4015 +4015 @@
     {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
     {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|Any CPU.Build.0 = Release|Any CPU
-    {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|x64.ActiveCfg = Release|x86
+    {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|x64.ActiveCfg = Release|x64
+    {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|x64.Build.0 = Release|x64
     {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|x86.ActiveCfg = Release|x86
     {1FC004FC-59AF-4249-B1B6-FF25873A20E4}.Release|x86.Build.0 = Release|x86
@@ -4047 +4048 @@
     {3127719F-110E-4558-8845-98559DBB422D}.Release|Any CPU.Build.0 = Release|Any CPU
     {3127719F-110E-4558-8845-98559DBB422D}.Release|x64.ActiveCfg = Release|x64
+    {3127719F-110E-4558-8845-98559DBB422D}.Release|x64.Build.0 = Release|x64
     {3127719F-110E-4558-8845-98559DBB422D}.Release|x86.ActiveCfg = Release|x86
     {3127719F-110E-4558-8845-98559DBB422D}.Release|x86.Build.0 = Release|x86