Changeset 12689 for branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing

Timestamp:

07/08/15 15:32:12 (9 years ago)

Author:

dglaser

Message:

#2388: Merged trunk into HiveStatistics branch

Location:

branches/HiveStatistics/sources

Files:

• . (modified) (1 prop)
• HeuristicLab.DataPreprocessing/3.4 (modified) (1 prop)
• HeuristicLab.DataPreprocessing/3.4/Implementations/DataGridContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Implementations/HistogramContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Implementations/PreprocessingChartContent.cs (modified) (2 diffs)
• HeuristicLab.DataPreprocessing/3.4/Implementations/SearchLogic.cs (modified) (5 diffs)
• HeuristicLab.DataPreprocessing/3.4/Implementations/StatisticsLogic.cs (modified) (7 diffs)
• HeuristicLab.DataPreprocessing/3.4/Interfaces/IDataGridContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs (modified) (2 diffs)

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/DataGridContent.cs

@@ -114 +114 @@
     }
 
-    public void DeleteRow(IEnumerable<int> rows) {
+    public void DeleteRows(IEnumerable<int> rows) {
       PreProcessingData.DeleteRowsWithIndices(rows);
       createRowNames();
+    }
+
+    public void DeleteColumn(int column) {
+      PreProcessingData.DeleteColumn(column);
     }
 

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/HistogramContent.cs

@@ -42 +42 @@
       set { this.classifierVariableIndex = value; }
     }
+    public bool IsDetailedChartViewEnabled { get; set; }
 
 

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/PreprocessingChartContent.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Analysis;
 using HeuristicLab.Common;
@@ -126 +127 @@
 
 
-    public ICheckedItemList<StringValue> CreateVariableItemList() {
+    public ICheckedItemList<StringValue> CreateVariableItemList(IEnumerable<string> checkedItems = null) {
       ICheckedItemList<StringValue> itemList = new CheckedItemList<StringValue>();
       foreach (string name in PreprocessingData.GetDoubleVariableNames()) {
-        itemList.Add(new StringValue(name), true);
+        var n = new StringValue(name);
+        itemList.Add(n, (checkedItems == null) ? true : checkedItems.Contains(name));
       }
       return new ReadOnlyCheckedItemList<StringValue>(itemList);

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/SearchLogic.cs

@@ -23 +23 @@
 using System.Collections;
 using System.Collections.Generic;
-using System.Linq;
 
 namespace HeuristicLab.DataPreprocessing {
   public class SearchLogic : ISearchLogic {
     private readonly ITransactionalPreprocessingData preprocessingData;
+    private readonly IFilterLogic filterLogic;
 
     private Dictionary<int, IList<int>> MissingValueIndicies { get; set; }
@@ -44 +44 @@
     }
 
-    public SearchLogic(ITransactionalPreprocessingData thePreprocessingData) {
+    public SearchLogic(ITransactionalPreprocessingData thePreprocessingData, IFilterLogic theFilterLogic) {
       preprocessingData = thePreprocessingData;
+      filterLogic = theFilterLogic;
 
       MissingValueIndicies = new Dictionary<int, IList<int>>();
       ValuesWithoutNaN = new Dictionary<int, IList>();
 
-      preprocessingData.Changed += preprocessingData_Changed;
+      preprocessingData.Changed += PreprocessingData_Changed;
+      filterLogic.FilterChanged += FilterLogic_FilterChanged;
     }
 
-    void preprocessingData_Changed(object sender, DataPreprocessingChangedEventArgs e)
-    {
+    void FilterLogic_FilterChanged(object sender, EventArgs e) {
+      //recalculate
+      for (int i = 0; i < Columns; i++) {
+        MissingValueIndicies.Remove(i);
+        ValuesWithoutNaN.Remove(i);
+      }
+    }
+
+    void PreprocessingData_Changed(object sender, DataPreprocessingChangedEventArgs e) {
       switch (e.Type) {
         case DataPreprocessingChangedEventType.DeleteColumn:
@@ -73 +82 @@
           ValuesWithoutNaN = new Dictionary<int, IList>();
           break;
-      } 
+      }
     }
 
@@ -97 +106 @@
 
     public IList<int> GetMissingValueIndices(int columnIndex) {
-      if (!MissingValueIndicies.ContainsKey(columnIndex)){        
-          if (preprocessingData.VariableHasType<double>(columnIndex)) {
-            MissingValueIndicies[columnIndex] = GetMissingValueIndices<double>(columnIndex);
-          } else if (preprocessingData.VariableHasType<string>(columnIndex)) {
-            MissingValueIndicies[columnIndex] = GetMissingValueIndices<string>(columnIndex);
-          } else if (preprocessingData.VariableHasType<DateTime>(columnIndex)) {
-            MissingValueIndicies[columnIndex] = GetMissingValueIndices<DateTime>(columnIndex);
-          } else {
-            throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
-          }
-      } 
+      if (!MissingValueIndicies.ContainsKey(columnIndex)) {
+        if (preprocessingData.VariableHasType<double>(columnIndex)) {
+          MissingValueIndicies[columnIndex] = GetMissingValueIndices<double>(columnIndex);
+        } else if (preprocessingData.VariableHasType<string>(columnIndex)) {
+          MissingValueIndicies[columnIndex] = GetMissingValueIndices<string>(columnIndex);
+        } else if (preprocessingData.VariableHasType<DateTime>(columnIndex)) {
+          MissingValueIndicies[columnIndex] = GetMissingValueIndices<DateTime>(columnIndex);
+        } else {
+          throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
+        }
+      }
+      return MissingValueIndicies[columnIndex];
+    }
 
-      return MissingValueIndicies[columnIndex];
-   }
     private IList<int> GetMissingValueIndices<T>(int columnIndex) {
       List<int> missingIndices = new List<int>();
-     
-      for(int row = 0; row < preprocessingData.Rows; ++row) {
+
+      for (int row = 0; row < preprocessingData.Rows; ++row) {
         if (IsMissingValue(columnIndex, row)) {
           missingIndices.Add(row);
@@ -123 +132 @@
     }
 
-    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection)
-    {
-      if (considerSelection) {     
-        var selectedRows =  preprocessingData.Selection[columnIndex];
-        
+    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection) {
+      if (considerSelection) {
+        var selectedRows = preprocessingData.Selection[columnIndex];
+
         List<T> values = new List<T>();
         foreach (var rowIdx in selectedRows) {

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/StatisticsLogic.cs

@@ -38 +38 @@
 
     public int GetColumnCount() {
-      return preprocessingData.Columns;
+      return searchLogic.Columns;
     }
 
     public int GetRowCount() {
-      return preprocessingData.Rows;
+      return searchLogic.Rows;
     }
 
@@ -48 +48 @@
       int count = 0;
 
-      for (int i = 0; i < preprocessingData.Columns; ++i) {
+      for (int i = 0; i < searchLogic.Columns; ++i) {
         if (preprocessingData.VariableHasType<double>(i)) {
           ++count;
@@ -57 +57 @@
 
     public int GetNominalColumnCount() {
-      return preprocessingData.Columns - GetNumericColumnCount();
+      return searchLogic.Columns - GetNumericColumnCount();
     }
 
     public int GetMissingValueCount() {
       int count = 0;
-      for (int i = 0; i < preprocessingData.Columns; ++i) {
+      for (int i = 0; i < searchLogic.Columns; ++i) {
         count += GetMissingValueCount(i);
       }
@@ -73 +73 @@
 
     public T GetMin<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
-      return preprocessingData.GetValues<T>(columnIndex, considerSelection).Min();
+      var min = default(T);
+      if (preprocessingData.VariableHasType<T>(columnIndex)) {
+        var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
+        if (values.Any()) {
+          min = values.Min();
+        }
+      }
+      return min;
     }
 
     public T GetMax<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
-      return preprocessingData.GetValues<T>(columnIndex, considerSelection).Max();
+      var max = default(T);
+      if (preprocessingData.VariableHasType<T>(columnIndex)) {
+        var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
+        if (values.Any()) {
+          max = values.Max();
+        }
+      }
+      return max;
     }
 
@@ -83 +97 @@
       double median = double.NaN;
       if (preprocessingData.VariableHasType<double>(columnIndex)) {
-        median = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Median();
+        var values = GetValuesWithoutNaN<double>(columnIndex, considerSelection);
+        if (values.Any()) {
+          median = values.Median();
+        }
       }
       return median;
@@ -91 +108 @@
       double avg = double.NaN;
       if (preprocessingData.VariableHasType<double>(columnIndex)) {
-        avg = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Average();
+        var values = GetValuesWithoutNaN<double>(columnIndex, considerSelection);
+        if (values.Any()) {
+          avg = values.Average();
+        }
       }
       return avg;
@@ -113 +133 @@
 
     public T GetMostCommonValue<T>(int columnIndex, bool considerSelection) {
-      var t = preprocessingData.GetValues<T>(columnIndex, considerSelection);
-      var t2 = t.GroupBy(x => x);
-      var t3 = t2.Select(g => g.Key);
-
-      return preprocessingData.GetValues<T>(columnIndex, considerSelection)
-                              .GroupBy(x => x)
+      var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
+      if (!values.Any())
+        return default(T);
+      return values.GroupBy(x => x)
                               .OrderByDescending(g => g.Count())
                               .Select(g => g.Key)

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Interfaces/IDataGridContent.cs

@@ -31 +31 @@
     IDictionary<int, IList<int>> Selection { get; set; }
 
-    void DeleteRow(IEnumerable<int> rows);
+    void DeleteRows(IEnumerable<int> rows);
+    void DeleteColumn(int column);
     bool Validate(string value, out string errorMessage, int columnIndex);
 

branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs

@@ -61 +61 @@
       SetTrainingAndTestPartition(problemData);
       // set the input variables to the correct checked state
-      var inputVariables = problemData.InputVariables.ToDictionary(x => x.Value, x => x);
-      foreach (var variable in oldProblemData.InputVariables) {
-        bool @checked = oldProblemData.InputVariables.ItemChecked(variable);
-        problemData.InputVariables.SetItemCheckedState(inputVariables[variable.Value], @checked);
+      var inputVariables = oldProblemData.InputVariables.ToDictionary(x => x.Value, x => x);
+      foreach (var variable in problemData.InputVariables) {
+        bool isChecked = oldProblemData.InputVariables.ItemChecked(inputVariables[variable.Value]);
+        problemData.InputVariables.SetItemCheckedState(variable, isChecked);
       }
 
@@ -77 +77 @@
 
     private IDataAnalysisProblemData CreateClassificationData(ClassificationProblemData oldProblemData) {
+      // target variable must be double and must exist in the new dataset
       var targetVariable = oldProblemData.TargetVariable;
-      // target variable must be double and must exist in the new dataset
-      return new ClassificationProblemData(ExportedDataset, GetDoubleInputVariables(targetVariable), targetVariable, Transformations);
+      var newProblemData = new ClassificationProblemData(ExportedDataset, GetDoubleInputVariables(targetVariable), targetVariable, Transformations);
+      newProblemData.PositiveClass = oldProblemData.PositiveClass;
+      return newProblemData;
     }