Changeset 14886 for trunk/sources/HeuristicLab.DataPreprocessing

Timestamp:

04/24/17 16:26:43 (8 years ago)

Author:

mkommend

Message:

#2778: Refactored and corrected shuffling in DataPreprocessing.

Location:

trunk/sources/HeuristicLab.DataPreprocessing/3.4

Files:

• HeuristicLab.DataPreprocessing-3.4.csproj (modified) (1 diff)
• Logic/ManipulationLogic.cs (modified) (3 diffs)
• Plugin.cs.frame (modified) (1 diff)

trunk/sources/HeuristicLab.DataPreprocessing/3.4/HeuristicLab.DataPreprocessing-3.4.csproj

@@ -167 +167 @@
       <Private>False</Private>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">
+      <Project>{F4539FB6-4708-40C9-BE64-0A1390AEA197}</Project>
+      <Name>HeuristicLab.Random-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Logic/ManipulationLogic.cs

@@ -24 +24 @@
 using System.Linq;
 using HeuristicLab.Data;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.DataPreprocessing {
@@ -81 +82 @@
     public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
       preprocessingData.InTransaction(() => {
-        Random r = new Random();
+        System.Random r = new System.Random();
 
         foreach (var column in cells) {
@@ -226 +227 @@
 
     public void Shuffle(bool shuffleRangesSeparately) {
-      Random random = new Random();
-      var ranges = new[] { preprocessingData.TestPartition, preprocessingData.TrainingPartition };
+      var random = new FastRandom();
+
       if (shuffleRangesSeparately) {
+        var ranges = new[] { preprocessingData.TestPartition, preprocessingData.TrainingPartition };
         preprocessingData.InTransaction(() => {
           // process all given ranges - e.g. TrainingPartition, TestPartition
           foreach (IntRange range in ranges) {
-            List<Tuple<int, int>> shuffledIndices = new List<Tuple<int, int>>();
-
-            // generate random indices used for shuffeling each column
-            for (int i = range.End - 1; i >= range.Start; --i) {
-              int rand = random.Next(range.Start, i);
-              shuffledIndices.Add(new Tuple<int, int>(i, rand));
-            }
-
-            ShuffleToIndices(shuffledIndices);
+            var indices = Enumerable.Range(0, preprocessingData.Rows).ToArray();
+            var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
+            for (int i = range.Start, j = 0; i < range.End; i++, j++)
+              indices[i] = shuffledIndices[j];
+
+            ReOrderToIndices(indices);
           }
         });
+
       } else {
         preprocessingData.InTransaction(() => {
-          var indices = ranges.SelectMany(x => Enumerable.Range(x.Start, x.Size)).ToList();
-          var shuffledIndices = indices.OrderBy(x => random.Next());
-          ShuffleToIndices(indices.Zip(shuffledIndices, (i, j) => new Tuple<int, int>(i, j)).ToList());
+          var indices = Enumerable.Range(0, preprocessingData.Rows);
+          var shuffledIndices = indices.Shuffle(random).ToArray();
+          ReOrderToIndices(shuffledIndices);
         });
       }
     }
 
-    public void ReOrderToIndices(IEnumerable<int> indices) {
-      List<Tuple<int, int>> indicesTuple = new List<Tuple<int, int>>();
-
-      for (int i = 0; i < indices.Count(); ++i) {
-        indicesTuple.Add(new Tuple<int, int>(i, indices.ElementAt(i)));
-      }
-
-      ReOrderToIndices(indicesTuple);
-    }
-
-    public void ReOrderToIndices(IList<System.Tuple<int, int>> indices) {
+    public void ReOrderToIndices(int[] indices) {
       preprocessingData.InTransaction(() => {
         for (int i = 0; i < preprocessingData.Columns; ++i) {
           if (preprocessingData.VariableHasType<double>(i)) {
-            reOrderToIndices<double>(i, indices);
+            ReOrderToIndices<double>(i, indices);
           } else if (preprocessingData.VariableHasType<string>(i)) {
-            reOrderToIndices<string>(i, indices);
+            ReOrderToIndices<string>(i, indices);
           } else if (preprocessingData.VariableHasType<DateTime>(i)) {
-            reOrderToIndices<DateTime>(i, indices);
-          }
-        }
-      });
-    }
-
-    public void ShuffleToIndices(IList<System.Tuple<int, int>> indices) {
-      preprocessingData.InTransaction(() => {
-        for (int i = 0; i < preprocessingData.Columns; ++i) {
-          if (preprocessingData.VariableHasType<double>(i)) {
-            ShuffleToIndices<double>(i, indices);
-          } else if (preprocessingData.VariableHasType<string>(i)) {
-            ShuffleToIndices<string>(i, indices);
-          } else if (preprocessingData.VariableHasType<DateTime>(i)) {
-            ShuffleToIndices<DateTime>(i, indices);
-          }
-        }
-      });
-    }
-
-    private void reOrderToIndices<T>(int columnIndex, IList<Tuple<int, int>> indices) {
-
+            ReOrderToIndices<DateTime>(i, indices);
+          }
+        }
+      });
+    }
+
+    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
       List<T> originalData = new List<T>(preprocessingData.GetValues<T>(columnIndex));
-
-      // process all columns equally
-      foreach (Tuple<int, int> index in indices) {
-        int originalIndex = index.Item1;
-        int replaceIndex = index.Item2;
+      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+
+      for (int i = 0; i < indices.Length; i++) {
+        int originalIndex = i;
+        int replaceIndex = indices[i];
 
         T replaceValue = originalData.ElementAt<T>(replaceIndex);
         preprocessingData.SetCell<T>(columnIndex, originalIndex, replaceValue);
-      }
-    }
-
-    private void ShuffleToIndices<T>(int columnIndex, IList<Tuple<int, int>> indices) {
-      // process all columns equally
-      foreach (Tuple<int, int> index in indices) {
-        int originalIndex = index.Item1;
-        int replaceIndex = index.Item2;
-
-        T tmp = preprocessingData.GetCell<T>(columnIndex, originalIndex);
-        T replaceValue = preprocessingData.GetCell<T>(columnIndex, replaceIndex);
-
-        preprocessingData.SetCell<T>(columnIndex, originalIndex, replaceValue);
-        preprocessingData.SetCell<T>(columnIndex, replaceIndex, tmp);
       }
     }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Plugin.cs.frame

@@ -34 +34 @@
   [PluginDependency("HeuristicLab.Collections", "3.3")]
   [PluginDependency("HeuristicLab.Data","3.3")]
-  [PluginDependency("HeuristicLab.Optimization", "3.3")]
+  [PluginDependency("HeuristicLab.Optimization", "3.3")]  
   [PluginDependency("HeuristicLab.Persistence", "3.3")]
   [PluginDependency("HeuristicLab.Problems.DataAnalysis","3.4")]
+  [PluginDependency("HeuristicLab.Random", "3.3")]
   public class HeuristicLabDataPreprocessingPlugin : PluginBase {
   }