Changeset 14543 for branches/HeuristicLab.DatastreamAnalysis

Timestamp:

01/04/17 18:38:52 (7 years ago)

Author:

jzenisek

Message:

#2719 enhanced Optimizer functionality; added results

Location:

branches/HeuristicLab.DatastreamAnalysis

Files:

• HeuristicLab.DatastreamAnalysis.Views/3.4/Plugin.cs (modified) (1 diff)
• HeuristicLab.DatastreamAnalysis.Views/3.4/Properties/AssemblyInfo.cs (modified) (1 diff)
• HeuristicLab.DatastreamAnalysis/3.4/Datastream.cs (modified) (1 diff)
• HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs (modified) (7 diffs)
• HeuristicLab.DatastreamAnalysis/3.4/Plugin.cs (modified) (1 diff)
• HeuristicLab.DatastreamAnalysis/3.4/Properties/AssemblyInfo.cs (modified) (1 diff)

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/Plugin.cs

@@ -23 +23 @@
 
 namespace HeuristicLab.DatastreamAnalysis.Views {
-  [Plugin("HeuristicLab.DatastreamAnalysis.Views", "3.4.14.14536")]
+  [Plugin("HeuristicLab.DatastreamAnalysis.Views", "3.4.14.14538")]
   [PluginFile("HeuristicLab.DatastreamAnalysis.Views-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.Collections", "3.3")]

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/Properties/AssemblyInfo.cs

@@ -54 +54 @@
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: AssemblyVersion("3.4.0.0")]
-[assembly: AssemblyFileVersion("3.4.14.14536")]
+[assembly: AssemblyFileVersion("3.4.14.14538")]

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Datastream.cs

@@ -181 +181 @@
     public Datastream() : base() {
       Parameters.Add(new ValueParameter<RegressionProblemData>(ProblemDataParameterName, "ProblemData for analysis with selected ensembles.", null));
-      Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 5)));
-      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(5)));
+      Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 20)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(20)));
       Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowStepWidthParameterName, "Sliding window step width", new IntValue(1)));
-      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementIntervalParameterName, "Sliding window movement interval (milliseconds)", new IntValue(2000)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementIntervalParameterName, "Sliding window movement interval (milliseconds)", new IntValue(500)));
       RegisterParameterEvents();
       InitializeState();

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs

@@ -144 +144 @@
       set { ((IntValue)Results["Sliding Window Movements"].Value).Value = value; }
     }
-    private double ResultsBestQuality {
-      get { return ((DoubleValue)Results["Best Quality"].Value).Value; }
-      set { ((DoubleValue)Results["Best Quality"].Value).Value = value; }
-    }
     private DataTable ResultsQualities {
       get { return ((DataTable)Results["Qualities"].Value); }
     }
 
-    private const string ResultsQualitiesR2 = "R²";
-    private const string ResultsQualitiesPearson = "Pearson";
+    private const string ResultsQualitiesMSE = "Mean squared error";
+    private const string ResultsQualitiesPR2 = "Pearson R²";
 
     private DataTable ResultsTargets {
@@ -161 +157 @@
     private const string ResultsTargetsReal = "Real";
 
+    private DataTable ResultsQualitiesBars {
+      get { return ((DataTable) Results["Ensembles"].Value); }
+    }
+
     protected void SetupResults() {
       Results.Clear();
 
       Results.Add(new Result("Sliding Window Movements", new IntValue(0)));
-      Results.Add(new Result("Best Quality", new DoubleValue(0)));
       Results.Add(new Result("Qualities", new DataTable("Qualities")));
       Results.Add(new Result("Targets", new DataTable("Targets")));
 
-      ResultsQualities.Rows.Add(new DataRow(ResultsQualitiesR2));
-      ResultsQualities.Rows.Add(new DataRow(ResultsQualitiesPearson));
+      ResultsQualitiesBars.Rows.Add(new DataRow("Ensembles"));
+
       ResultsTargets.Rows.Add(new DataRow(ResultsTargetsReal));
       foreach (var ensemble in Ensembles) {
+        // targets table
         ResultsTargets.Rows.Add(new DataRow(ensemble.Name));
+
+        // qualities (series)
+        //ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesMSE));
+        ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesPR2));
+
+        // qualities (bars)
+        //ResultsQualitiesBars.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesPR2));
+        //ResultsQualitiesBars.Rows["Ensembles"].Values.
       }
     }
@@ -180 +188 @@
 
     #region constructors, cloner,...
-
     public DatastreamAnalysisOptimizer() : base() {
       name = "Datastream Analysis";
@@ -248 +255 @@
       if (prepared) {
         SetupResults();
-        //prepared = false;
+        Datastream.InitializeState();
       }
 
@@ -273 +280 @@
           OnPaused();
         }
-        //if(!Datastream.SlidingWindowMovementPossible) OnStopped();
-        //else OnPaused();
       });
-
     }
 
@@ -361 +365 @@
 
     protected void Run(CancellationToken cancellationToken) {
-      // init algorithm
-      var problemData = Datastream.ProblemData;
-      var targetVarName = problemData.TargetVariable;
-      var activeVariables = problemData.AllowedInputVariables;
 
       if (prepared) {
-        // replay datastream until FitnessPartition.End
-        //for (int i = 0; i < Datastream.FitnessPartition.End; i++) {
-        //  var val = problemData.Dataset.GetDoubleValue(targetVarName, i);
-        //  ResultsTargets.Rows[ResultsTargetsReal].Values.Add(val);
-        //}
-        //replayedIndex = Datastream.FitnessPartition.End;
         replayedIndex = 0;
         prepared = false;
       }
 
-
-      Random rnd = new Random();
-
       try {
+
+        // play and evaluate initial window
+        PlayDatastream();
+        if(Datastream.SlidingWindowEvaluationPossible) Evaluate();
+        replayedIndex = Datastream.FitnessPartition.End;
+
         do {
-          do {
+          while(Datastream.SlidingWindowMovementPossible) {
             cancellationToken.ThrowIfCancellationRequested();
 
+            // perform (delayed) window movement
             Thread.Sleep(Datastream.SlidingWindowMovementInterval.Value);
             Datastream.MoveSlidingWindow();
-
-            if (Datastream.SlidingWindowEvaluationPossible) {
-              foreach (var ensemble in Ensembles) {
-                // TODO do the window evaluation
-                double estimatedVal = problemData.Dataset.GetDoubleValue(targetVarName, replayedIndex)
-                                      * (1.0 + (((double)rnd.Next(0, 30)) / 100));
-                for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
-                  ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedVal);
-                }
-              }
-            }
-
-            // replay datastream until FitnessPartition.End
-            for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
-              var val = problemData.Dataset.GetDoubleValue(targetVarName, i);
-              ResultsTargets.Rows[ResultsTargetsReal].Values.Add(val);
-            }
+            ResultsSlidingWindowMovements++;
+
+            // play and evaluate the moved window
+            PlayDatastream();
+            if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
             replayedIndex = Datastream.FitnessPartition.End;
-
-
-            // TODO: collect results and display them
-
-            int exp = rnd.Next(100);
-            ResultsQualities.Rows[ResultsQualitiesR2].Values.Add(exp);
-            ResultsQualities.Rows[ResultsQualitiesPearson].Values.Add((double)exp / 10);
-            ResultsSlidingWindowMovements++;
-            ResultsBestQuality = (double)exp / 42;
-
-          } while (Datastream.SlidingWindowMovementPossible);
+          }
         } while (Datastream.UpdateAvailable);
         finished = true;
@@ -426 +402 @@
         // reset everything
         //Prepare(true);
+      }
+    }
+
+    private void PlayDatastream() {
+      var problemData = Datastream.ProblemData;
+      var targetVarName = problemData.TargetVariable;
+
+      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
+        var realValue = problemData.Dataset.GetDoubleValue(targetVarName, i);
+        ResultsTargets.Rows[ResultsTargetsReal].Values.Add(realValue);
+      }
+    }
+
+    private void Evaluate() {
+      var problemData = Datastream.ProblemData;
+      var dataset = problemData.Dataset;
+      var targetVarName = problemData.TargetVariable;
+
+      var realRows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size).ToList();
+      var realValues = dataset.GetDoubleValues(targetVarName, realRows);
+
+      foreach (var ensemble in Ensembles) {
+
+        var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
+        var estimatedValuesPerModelPerRow = ensemble.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
+        var estimatedValues = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
+        var averageEstimatedValuesPerModel = estimatedValuesPerModelPerRow.Select(x => x.Average()); // per model
+        var averageEstimatedValue = averageEstimatedValuesPerModel.Average();
+
+        // determine quality
+        var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
+        OnlineCalculatorError error;
+        var pR = OnlinePearsonsRCalculator.Calculate(estimatedValues, realValues, out error);
+        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
+
+        DataRow qualitiesBarsRow = ResultsQualitiesBars.Rows[ensemble.Name + " - " + ResultsQualitiesPR2];
+
+        for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
+          ResultsTargets.Rows[ensemble.Name].Values.Add(averageEstimatedValue);
+
+          //ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesMSE].Values.Add(mse);
+          ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesPR2].Values.Add(pR2);
+        }
       }
     }

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Plugin.cs

@@ -23 +23 @@
 
 namespace HeuristicLab.DatastreamAnalysis {
-  [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14536")]
+  [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14538")]
   [PluginFile("HeuristicLab.DatastreamAnalysis-3.4.dll", PluginFileType.Assembly)]
   [PluginDependency("HeuristicLab.Collections", "3.3")]

branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Properties/AssemblyInfo.cs

@@ -54 +54 @@
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: AssemblyVersion("3.4.0.0")]
-[assembly: AssemblyFileVersion("3.4.14.14536")]
+[assembly: AssemblyFileVersion("3.4.14.14538")]