Changeset 8294 for branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4

Timestamp:

07/16/12 17:23:16 (12 years ago)

Author:

sforsten

Message:

#1292:

• SpearmansRankCorrelationCoefficientCalculator now uses the alglib function
• strings in ExtendedHeatMap have been made constant

Location:

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4

Files:

• HeuristicLab.Problems.DataAnalysis-3.4.csproj (modified) (1 diff)
• Implementation/ExtendedHeatMap.cs (modified) (2 diffs)
• OnlineCalculators/SpearmansRankCorrelationCoefficientCalculator.cs (modified) (1 diff)

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

@@ -93 +93 @@
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ALGLIB-3.5.0, Version=3.5.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <Private>False</Private>
+    </Reference>
     <Reference Include="HeuristicLab.Analysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <Private>False</Private>

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/ExtendedHeatMap.cs

@@ -57 +57 @@
     }
 
+    private const string PrearsonsRSquared = "Pearsons R Squared";
+    private const string HoeffdingsDependence = "Hoeffdings Dependence";
+    private const string SpearmansRank = "Spearmans Rank";
     public IEnumerable<string> CorrelationCalculators {
-      get { return new List<string>() { "OnlinePearsonsRSquaredCalculator", "HoeffdingsDependenceCalculator", "SpearmansRankCorrelationCoefficientCalculator" }; }
+      get { return new List<string>() { PrearsonsRSquared, HoeffdingsDependence, SpearmansRank }; }
     }
 
+    private const string AllSamples = "All Samples";
+    private const string TrainingSamples = "Training Samples";
+    private const string TestSamples = "Test Samples";
     public IEnumerable<string> Partitions {
-      get { return new List<string>() { "All Samples", "Training Partition", "Test Partition" }; }
+      get { return new List<string>() { AllSamples, TrainingSamples, TestSamples }; }
     }
 
@@ -112 +118 @@
           IEnumerable<double> var1 = dataset.GetDoubleValues(doubleVariableNames[i]);
           IEnumerable<double> var2 = dataset.GetDoubleValues(doubleVariableNames[j]);
-          if (partition.Equals(Partitions.ElementAt(1))) {
+          if (partition.Equals(TrainingSamples)) {
             var1 = var1.Skip(problemData.TrainingPartition.Start).Take(problemData.TrainingPartition.End - problemData.TrainingPartition.Start);
             var2 = var2.Skip(problemData.TrainingPartition.Start).Take(problemData.TrainingPartition.End - problemData.TrainingPartition.Start);
-          } else if (partition.Equals(Partitions.ElementAt(2))) {
+          } else if (partition.Equals(TestSamples)) {
             var1 = var1.Skip(problemData.TestPartition.Start).Take(problemData.TestPartition.End - problemData.TestPartition.Start);
             var2 = var2.Skip(problemData.TestPartition.Start).Take(problemData.TestPartition.End - problemData.TestPartition.Start);
           }
 
-          if (calc.Equals(CorrelationCalculators.ElementAt(1))) {
+          if (calc.Equals(HoeffdingsDependence)) {
             elements[i, j] = HoeffdingsDependenceCalculator.Calculate(var1, var2, out error);
-          } else if (calc.Equals(CorrelationCalculators.ElementAt(2))) {
+          } else if (calc.Equals(SpearmansRank)) {
             elements[i, j] = SpearmansRankCorrelationCoefficientCalculator.Calculate(var1, var2, out error);
           } else {

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/SpearmansRankCorrelationCoefficientCalculator.cs

@@ -28 +28 @@
 
     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
-      double rs = Math.Abs(Spear(originalValues, estimatedValues, out errorState));
-      if (errorState != OnlineCalculatorError.None) return double.NaN;
-      return rs;
-    }
-
-    /// <summary>
-    /// Calculates Spearmans Rank Correlation Coefficient. Source: Numerical Recipes in C.
-    /// </summary>
-    private static double Spear(IEnumerable<double> xs, IEnumerable<double> ys, out OnlineCalculatorError errorState) {
-      double[] xsArr = xs.ToArray();
-      double[] ysArr = ys.ToArray();
-      if (xsArr.Length != ysArr.Length) throw new ArgumentException("The number of elements in xs and ys does not match");
-
-      int n = xsArr.Length;
-      Array.Sort(xsArr, ysArr);
-      double sf = CRank(xsArr);
-      Array.Sort(ysArr, xsArr);
-      double sg = CRank(ysArr);
-
-      double d = 0.0;
-      for (int j = 0; j < n; j++) //Sum the squared difference of ranks.
-        d += Math.Pow(xsArr[j] - ysArr[j], 2);
-
-      double en, en3n, aved, fac, rs; // vard, zd, t, df, probd, probrs;
-      en = n;
-      en3n = en * en * en - en;
-      aved = en3n / 6.0 - (sf + sg) / 12.0;
-      fac = (1.0 - sf / en3n) * (1.0 - sg / en3n);
-      //vard = ((en - 1.0) * en * en * Math.Pow(en + 1.0, 2) / 36.0) * fac;
-      //zd = (d - aved) / Math.Sqrt(vard);
-      //probd = erfcc(Math.Abs(zd) / 1.4142136);
-      rs = (1.0 - (6.0 / en3n) * (d + (sf + sg) / 12.0)) / Math.Sqrt(fac);
-      //fac = (rs + 1.0) * (1.0 - rs);
-      //if (fac > 0.0) {
-      //  t = rs * Math.Sqrt((en - 2.0) / fac);
-      //  df = en - 2.0;
-      //  probrs = betai(0.5 * df, 0.5, df / (df + t * t));
-      //} else {
-      //  probrs = 0.0;
-      //}
+      double rs = Math.Abs(alglib.basestat.spearmancorr2(originalValues.ToArray(), estimatedValues.ToArray(), originalValues.Count()));
       errorState = OnlineCalculatorError.None;
       return rs;
     }
-
-    /// <summary>
-    /// Calculates midranks. Source: Numerical Recipes in C.
-    /// </summary>
-    /// <param name="w">Sorted array of elements, replaces the elements by their rank, including midranking of ties</param>
-    /// <returns></returns>
-    private static double CRank(double[] w) {
-      int i = 0;
-      int n = w.Length;
-      double s = 0.0;
-      double t;
-      while (i < n - 1) {
-        if (w[i + 1] > w[i]) {    // w[i+1] must be larger or equal w[i] as w must be sorted
-          // not a tie
-          w[i] = i + 1;
-          i++;
-        } else {
-          int j;
-          for (j = i + 1; j < n && w[j] <= w[i]; j++) ; // how far does it go (<= effectively means == as w must be sorted)
-          double rank = 0.5 * (i + j - 1);
-          int k;
-          for (k = i; k <= j - 1; k++) w[k] = rank; // set the rank for all tied entries
-          t = j - i;
-          s += t * t * t - t;
-          i = j;
-        }
-      }
-
-      if (i == n - 1) w[n - 1] = n - 1;   // if the last element was not tied, this is its rank
-      return s;
-    }
   }
 }