Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
03/02/17 14:10:57 (7 years ago)
Author:
jzenisek
Message:

#2719 implemented ensemble model rating by introducing the new type RatedEnsembleModel; introduced performance indicator calculation in results;

Location:
branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4
Files:
2 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DataBar.cs

    r14588 r14710  
    2828using HeuristicLab.Common;
    2929using HeuristicLab.Core;
     30using HeuristicLab.Data;
    3031using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    3132
    … …  
    3839  public class DataBar : NamedItem, INotifyPropertyChanged {
    3940    private double value;
     41    public static Color DefaultBarColor = Color.FromArgb(65, 140, 240);
    4042
    4143    public double Value {
    … …  
    5052    }
    5153
    52     private double threshold;
     54    private double thresholdLowerBound;
    5355
    54     public double Threshold {
    55       get { return threshold; }
     56    public double ThresholdLowerBound {
     57      get { return thresholdLowerBound; }
    5658      set {
     59        if (this.value != value) {
     60          this.value = value;
     61          OnThresholdPropertyChanged("Threshold");
     62        }
     63      }
     64    }
     65
     66    private double thresholdUpperBound;
     67
     68    public double ThresholdUpperBound
     69    {
     70      get { return thresholdUpperBound; }
     71      set
     72      {
    5773        if (this.value != value) {
    5874          this.value = value;
    … …  
    6884      set { this.value = value; }
    6985    }
    70     [Storable(Name = "Threshold")]
    71     private double StorableThreshold {
    72       get { return threshold; }
    73       set { this.threshold = value; }
     86    [Storable(Name = "ThresholdLowerBound")]
     87    private double StorableThresholdLowerBound {
     88      get { return thresholdLowerBound; }
     89      set { this.thresholdLowerBound = value; }
     90    }
     91    [Storable(Name = "ThresholdUpperBound")]
     92    private double StorableThresholdUpperBound
     93    {
     94      get { return thresholdUpperBound; }
     95      set { this.thresholdUpperBound = value; }
    7496    }
    7597    #endregion
    … …  
    109131    protected DataBar(DataBar original, Cloner cloner) : base(original, cloner) {
    110132      this.value = original.value;
    111       this.threshold = original.threshold;
     133      this.thresholdLowerBound = original.thresholdLowerBound;
    112134      this.barColor = original.barColor;
    113135      this.thresholdColor = original.thresholdColor;
    … …  
    115137
    116138    public DataBar() : this("DataBar") {
    117       barColor = Color.Empty;
    118       thresholdColor = Color.Red;
     139      barColor = DefaultBarColor;
     140      thresholdColor = Color.Black;
    119141    }
    120142
    121143    public DataBar(string name) : base(name) {
    122144      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    123       barColor = Color.Empty;
    124       thresholdColor = Color.Red;
     145      barColor = DefaultBarColor;
     146      thresholdColor = Color.Black;
    125147    }
    126148
    127     public DataBar(string name, double threshold) : base(name) {
     149    public DataBar(string name, double thresholdLowerBound, double thresholdUpperBound) : base(name) {
    128150      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    129       this.threshold = threshold;
    130       barColor = Color.Empty;
    131       thresholdColor = Color.Red;
     151      this.thresholdLowerBound = thresholdLowerBound;
     152      this.thresholdUpperBound = thresholdUpperBound;
     153      barColor = DefaultBarColor;
     154      thresholdColor = Color.Black;
    132155    }
    133156
    134157    public DataBar(string name, string description) : base(name, description) {
    135158      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    136       barColor = Color.Empty;
    137       thresholdColor = Color.Red;
     159      barColor = DefaultBarColor;
     160      thresholdColor = Color.Black;
    138161    }
    139162

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Datastream.cs

    r14543 r14710  
    4242    protected const string SlidingWindowSizeParameterName = "SlidingWindowSize";
    4343    protected const string SlidingWindowStepWidthParameterName = "SlidingWindowStepWidth";
    44     protected const string SlidingWindowMovementIntervalParameterName = "SlidingWindowMovementInterval";
     44    protected const string SlidingWindowMovementDelayParameterName = "SlidingWindowMovementDelay";
    4545    protected const string FitnessPartitionParameterName = "FitnessPartition";
     46    protected const string SlidingWindowSonarRatioParameterName = "SlidingWindowSonarRatio";
    4647
    4748    #region parameter properites
    … …  
    6364    }
    6465
    65     public IValueParameter<IntValue> SlidingWindowMovementIntervalParameter {
    66       get { return (IValueParameter<IntValue>) Parameters[SlidingWindowMovementIntervalParameterName]; }
     66    public IValueParameter<IntValue> SlidingWindowMovementDelayParameter {
     67      get { return (IValueParameter<IntValue>) Parameters[SlidingWindowMovementDelayParameterName]; }
     68    }
     69
     70    public IValueParameter<DoubleValue> SlidingWindowSonarRatioParameter {
     71      get { return (IValueParameter<DoubleValue>) Parameters[SlidingWindowSonarRatioParameterName]; }
    6772    }
    6873
    … …  
    7883      get { return ProblemDataParameter.Value; }
    7984      set {
    80         if(value == null) throw new ArgumentNullException("problemData", "The provided value for the problemData is null.");
     85        if(value == null) throw new ArgumentNullException("ProblemData", "The provided value for problemData is null.");
    8186        ProblemDataParameter.Value = value;
    8287        //OnProblemDataChanged();
    … …  
    100105    }
    101106
    102     public IntValue SlidingWindowMovementInterval {
    103       get { return SlidingWindowMovementIntervalParameter.Value; }
    104       set { SlidingWindowMovementIntervalParameter.Value = value; }
     107    public IntValue SlidingWindowMovementDelay {
     108      get { return SlidingWindowMovementDelayParameter.Value; }
     109      set { SlidingWindowMovementDelayParameter.Value = value; }
     110    }
     111
     112    public DoubleValue SlidingWindowSonarRatio {
     113      get { return SlidingWindowSonarRatioParameter.Value; }
     114      set { SlidingWindowSonarRatioParameter.Value = value; }
    105115    }
    106116
    … …  
    181191    public Datastream() : base() {
    182192      Parameters.Add(new ValueParameter<RegressionProblemData>(ProblemDataParameterName, "ProblemData for analysis with selected ensembles.", null));
    183       Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 20)));
    184       Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(20)));
     193      Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 100)));
     194      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(100)));
    185195      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowStepWidthParameterName, "Sliding window step width", new IntValue(1)));
    186       Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementIntervalParameterName, "Sliding window movement interval (milliseconds)", new IntValue(500)));
     196      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementDelayParameterName, "Sliding window movement delay interval (milliseconds)", new IntValue(0)));
     197      Parameters.Add(new FixedValueParameter<DoubleValue>(SlidingWindowSonarRatioParameterName, "Sliding window sonar ratio", new DoubleValue(0.25)));
    187198      RegisterParameterEvents();
    188199      InitializeState();

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs

    r14588 r14710  
    7474    }
    7575
     76    private double lastStateValue;
     77    private double pR2Sum;
     78    private int detectionCount;
     79    private int evaluationCount;
     80    private double exclusivitySum;
     81    private double accuracySum;
     82
    7683    private CancellationTokenSource cancellationTokenSource;
    7784    private bool stopPending;
    … …  
    100107    }
    101108
    102 
    103109    [Storable]
    104     private IItemList<RegressionEnsembleModel> ensembles;
    105 
    106     public IItemList<RegressionEnsembleModel> Ensembles {
     110    private IItemList<RatedEnsembleModel> ensembles;
     111
     112    public IItemList<RatedEnsembleModel> Ensembles {
    107113      get { return ensembles; }
    108114      set {
    109115        if (value == null || value == ensembles)
    110116          return;
    111         if(!(value is IRegressionEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
     117        if (!(value is IRatedEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
    112118        DeregisterEnsembleEvents();
    113119        ensembles = value;
    … …  
    145151    }
    146152    private DataTable ResultsQualities {
    147       get { return ((DataTable)Results["Qualities"].Value); }
     153      get { return ((DataTable)Results["Qualities 1"].Value); }
     154    }
     155
     156    private DataTable ResultsPeakQualities {
     157      get { return ((DataTable) Results["Qualities 2"].Value); }
    148158    }
    149159
    … …  
    155165    }
    156166
     167    private DataTable ResultsStateDetection {
     168      get { return ((DataTable) Results["State Detection"].Value);}
     169    }
     170
    157171    private const string ResultsTargetsReal = "Real";
    158172
    159     private DataBarSet ResultsQualitiesBars {
    160       get { return (DataBarSet) Results["Ensemble Comparison"].Value; }
    161     }
     173    private DataBarSet ResultsSWQualitiesBars {
     174      get { return (DataBarSet) Results["Ensemble Comparison [SW-Quality]"].Value; }
     175    }
     176
     177    private DataBarSet ResultsSWVotingBars {
     178      get { return (DataBarSet) Results["Ensemble Comparison [SW-Voting]"].Value; }
     179    }
     180
     181    private DataBarSet ResultsSonarQualitiesBars {
     182      get { return (DataBarSet)Results["Ensemble Comparison [Sonar-Quality]"].Value; }
     183    }
     184
     185    private DataBarSet ResultsSWPeakQualitiesBars {
     186      get { return (DataBarSet)Results["Ensemble Comparison [SW-PeakQuality]"].Value; }
     187    }
     188
     189    private double ResultsMeanPR2 {
     190      get { return ((DoubleValue) Results["Mean PR2"].Value).Value; }
     191      set { ((DoubleValue)Results["Mean PR2"].Value).Value = value; }
     192    }
     193
     194    private double ResultsDetectionAccuracy {
     195      get { return ((DoubleValue)Results["Detection Accuracy"].Value).Value; }
     196      set { ((DoubleValue)Results["Detection Accuracy"].Value).Value = value; }
     197    }
     198
     199    private double ResultsDetectionExclusivity {
     200      get { return ((DoubleValue)Results["Detection Exclusivity"].Value).Value; }
     201      set { ((DoubleValue)Results["Detection Exclusivity"].Value).Value = value; }
     202    }
     203
     204    //private int ResultsTrueDetectionCount {
     205    //  get { return ((IntValue)Results["True Detection Count"].Value).Value; }
     206    //  set { ((IntValue)Results["True Detection Count"].Value).Value = value; }
     207    //}
     208
     209    //private int ResultsFalseDetectionCount {
     210    //  get { return ((IntValue)Results["False Detection Count"].Value).Value; }
     211    //  set { ((IntValue)Results["False Detection Count"].Value).Value = value; }
     212    //}
    162213
    163214    protected void SetupResults() {
     215      evaluationCount = 0;
     216      lastStateValue = 0.0;
     217      pR2Sum = 0.0;
     218      exclusivitySum = 0.0;
     219      accuracySum = 0.0;
     220      detectionCount = 0;
    164221      Results.Clear();
    165 
    166222      Results.Add(new Result("Sliding Window Movements", new IntValue(0)));
    167       Results.Add(new Result("Qualities", new DataTable("Pearson R²")));
     223      Results.Add(new Result("Qualities 1", new DataTable("Average Pearson R²")));
     224      Results.Add(new Result("Qualities 2", new DataTable("Peak Pearson R²")));
    168225      Results.Add(new Result("Targets", new DataTable("Targets")));
    169       Results.Add(new Result("Ensemble Comparison", new DataBarSet("Ensemble Comparison")));
     226      Results.Add(new Result("State Detection", new DataTable("State Detection")));
     227      Results.Add(new Result("Ensemble Comparison [SW-Quality]", new DataBarSet("Ensemble Comparison [SW-Quality]")));
     228      Results.Add(new Result("Ensemble Comparison [SW-Voting]", new DataBarSet("Ensemble Comparison [SW-Voting]")));
     229      Results.Add(new Result("Ensemble Comparison [Sonar-Quality]", new DataBarSet("Ensemble Comparison [Sonar-Quality]")));
     230      Results.Add(new Result("Ensemble Comparison [SW-PeakQuality]", new DataBarSet("Ensemble Comparison [SW-PeakQuality]")));
     231      Results.Add(new Result("Mean PR2", new DoubleValue()));
     232      Results.Add(new Result("Detection Accuracy", new DoubleValue()));
     233      Results.Add(new Result("Detection Exclusivity", new DoubleValue()));
    170234
    171235      ResultsTargets.Rows.Add(new DataRow(ResultsTargetsReal));
    … …  
    174238        ResultsTargets.Rows.Add(new DataRow(ensemble.Name));
    175239
     240        ResultsStateDetection.Rows.Add(new DataRow(ensemble.Name));
     241
    176242        // qualities (series)
    177243        //ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesMSE));
    178         ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesPR2));
     244        ResultsQualities.Rows.Add(new DataRow(ensemble.Name));
     245
     246        ResultsPeakQualities.Rows.Add(new DataRow(ensemble.Name));
    179247
    180248        // qualities (bars)
    181         ResultsQualitiesBars.Bars.Add(new DataBar(ensemble.Name, 0.9));
     249        ResultsSWQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
     250
     251        // voting (bars)
     252        ResultsSWVotingBars.Bars.Add(new DataBar(ensemble.Name, ensemble.ConfidenceThreshold.Start, ensemble.ConfidenceThreshold.End));
     253
     254        // sonar quality (bars)
     255        ResultsSonarQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
     256
     257        // quality peaks (bars)
     258        ResultsSWPeakQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
    182259      }
    183260    }
    … …  
    190267      log = new Log();
    191268      results = new ResultCollection();
    192       ensembles = new ItemList<RegressionEnsembleModel>();
     269      ensembles = new ItemList<RatedEnsembleModel>();
    193270      datastream = new Datastream();
    194271      runsCounter = 0;
    … …  
    210287      log = cloner.Clone(original.log);
    211288      results = cloner.Clone(original.results);
    212       ensembles = (ItemList<RegressionEnsembleModel>) original.Ensembles.Clone(cloner);
     289      ensembles = (ItemList<RatedEnsembleModel>) original.Ensembles.Clone(cloner);
    213290      datastream = (Datastream) original.Datastream.Clone(cloner);
    214291      runsCounter = original.runsCounter;
    … …  
    373450        // play and evaluate initial window
    374451        PlayDatastream();
    375         if(Datastream.SlidingWindowEvaluationPossible) Evaluate();
     452        lastStateValue = Datastream.ProblemData.Dataset.GetDoubleValue("h", replayedIndex);
     453        if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
    376454        replayedIndex = Datastream.FitnessPartition.End;
    377455
    … …  
    381459
    382460            // perform (delayed) window movement
    383             Thread.Sleep(Datastream.SlidingWindowMovementInterval.Value);
     461            Thread.Sleep(Datastream.SlidingWindowMovementDelay.Value);
    384462            Datastream.MoveSlidingWindow();
    385463            ResultsSlidingWindowMovements++;
    … …  
    412490      }
    413491    }
    414 
     492   
    415493    private void Evaluate() {
     494      evaluationCount++;
    416495      var problemData = Datastream.ProblemData;
    417496      var dataset = problemData.Dataset;
    418497      var targetVarName = problemData.TargetVariable;
    419498
    420       var realRows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size).ToList();
    421       var realValues = dataset.GetDoubleValues(targetVarName, realRows);
    422 
     499      var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
     500      var realValues = dataset.GetDoubleValues(targetVarName, rows);
     501
     502      int sonarSize = (int)(Datastream.FitnessPartition.Size * Datastream.SlidingWindowSonarRatio.Value);
     503      var sonarStart = Datastream.FitnessPartition.End - sonarSize;
     504      var sonarRows = Enumerable.Range(sonarStart, sonarSize);
     505      var sonarRealValues = dataset.GetDoubleValues(targetVarName, sonarRows);
     506
     507      var winningEnsembleName = ensembles[0].Name;
     508      var winningEnsemblePR2 = 0.0;
     509      var winningEnsembleIdx = 0;
     510      var ec = 0;
     511      bool stateChangeInStep = false;
     512      var curDetectionCount = 0;
     513      var changeDetectionCount = 0;
     514      double curStateValue = 0.0;
    423515      foreach (var ensemble in Ensembles) {
    424516
    425         var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
    426         var estimatedValuesPerModelPerRow = ensemble.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
    427         var estimatedValues = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
     517        var sonarEstimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, sonarRows).ToArray());
     518        var sonarEstimatedValuesPerRow = Enumerable.Range(0, sonarSize).Select(r => sonarEstimatedValuesPerModelPerRow.Select(m => m[r]).Average()).ToArray();
     519
     520        var estimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
     521        var estimatedValuesPerRow = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
    428522        var averageEstimatedValuesPerModel = estimatedValuesPerModelPerRow.Select(x => x.Average()); // per model
    429523        var averageEstimatedValue = averageEstimatedValuesPerModel.Average();
    430524
    431         // determine quality
    432         var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
     525        // calulate quality
     526        //var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
    433527        OnlineCalculatorError error;
    434         var pR = OnlinePearsonsRCalculator.Calculate(estimatedValues, realValues, out error);
     528        var pR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerRow, realValues, out error);
    435529        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
    436530
     531        var sonarPR = OnlinePearsonsRCalculator.Calculate(sonarEstimatedValuesPerRow, sonarRealValues, out error);
     532        var sonarPR2 = error == OnlineCalculatorError.None ? sonarPR * sonarPR : 0.0;
     533   
     534        var estimatedValuesPerModelPerRowArr = estimatedValuesPerModelPerRow.ToArray();
     535        var curPR = 0.0;
     536       
     537        var peakPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[0], realValues, out error);
     538        for (int i = 1; i < estimatedValuesPerModelPerRow.Count(); i++) {
     539          curPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[i], realValues, out error);
     540          if (curPR > peakPR && error == OnlineCalculatorError.None) {
     541            peakPR = curPR;
     542          }
     543        }
     544        var peakPR2 = peakPR * peakPR;
     545
     546        // calculate confidence
     547        var cm1 = CalulateEnsembleConfidenceCM1(ensemble, estimatedValuesPerModelPerRow, realValues);
     548
    437549        int replayAmount = Datastream.FitnessPartition.End - replayedIndex;
    438         int replayCount = estimatedValues.Length - replayAmount;
     550        int replayCount = estimatedValuesPerRow.Length - replayAmount;
    439551
    440552        for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
    441553          //ResultsTargets.Rows[ensemble.Name].Values.Add(averageEstimatedValue);
    442           ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValues[replayCount]);
     554          ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValuesPerRow[replayCount]);
    443555          replayCount++;
    444556
    445557          //ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesMSE].Values.Add(mse);
    446           ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesPR2].Values.Add(pR2);
    447           ResultsQualitiesBars.Bars[ensemble.Name].Value = pR2;
    448         }
    449       }
     558          ResultsQualities.Rows[ensemble.Name].Values.Add(pR2);
     559          ResultsPeakQualities.Rows[ensemble.Name].Values.Add(peakPR2);
     560
     561          double amp = (pR2 > ensemble.QualityThreshold.Start) ? 1.0 : 0.0;
     562          ResultsStateDetection.Rows[ensemble.Name].Values.Add(amp);   
     563        }
     564        ResultsSWQualitiesBars.Bars[ensemble.Name].Value = pR2;
     565        ResultsSonarQualitiesBars.Bars[ensemble.Name].Value = sonarPR2;
     566        ResultsSWVotingBars.Bars[ensemble.Name].Value = cm1;
     567        ResultsSWPeakQualitiesBars.Bars[ensemble.Name].Value = peakPR2;
     568
     569        if (pR2 > ensemble.QualityThreshold.Start)
     570          curDetectionCount++;
     571
     572        if (pR2 > winningEnsemblePR2) {
     573          winningEnsemblePR2 = pR2;
     574          winningEnsembleName = ensemble.Name;
     575          winningEnsembleIdx = ec;
     576        }
     577        ec++;
     578      }
     579
     580      // quality meassures
     581      // =================================================
     582
     583      if (winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     584        // Mean PR2
     585        pR2Sum += winningEnsemblePR2;
     586        detectionCount++;
     587        ResultsMeanPR2 = pR2Sum/detectionCount;
     588
     589        // detection exclusivity --> 1.0 - (x - 1) * (1.0 / n)
     590        var fraction = 1.0 / ensembles.Count;
     591        var curExclusivity = 1.0 - (curDetectionCount - 1) * fraction;
     592        exclusivitySum += curExclusivity;
     593        ResultsDetectionExclusivity = exclusivitySum / detectionCount;
     594      }
     595
     596      double replCount = 0.0;
     597      double accCount = 0.0;
     598      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
     599        replCount++;
     600        curStateValue = problemData.Dataset.GetDoubleValue("h", i);
     601        if (curStateValue <= 0.5 && winningEnsembleIdx == 0 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     602          accCount++;
     603        } else if (curStateValue > 0.5 && winningEnsembleIdx == 1 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     604          accCount++;
     605        }
     606      }
     607      accuracySum += accCount / replCount;
     608      ResultsDetectionAccuracy = accuracySum / evaluationCount;
     609
     610      //if (accCount / replCount > 1.0) {
     611      //  Console.WriteLine();
     612      //}
     613
     614      //if(accuracySum / evaluationCount > 1.0) {
     615      //  Console.WriteLine();
     616      //}
     617
     618      //if (accCount < replCount && !accDiscount) accDiscount = true;
     619      //if (accDiscount && accuracySum/evaluationCount >= 1.0) {
     620      //  Console.WriteLine();
     621      //}
     622
     623      lastStateValue = curStateValue;
     624
     625      // detection of state change
     626      //for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
     627      //  curStateValue = problemData.Dataset.GetDoubleValue("h", i);
     628      //  if (lastStateValue > curStateValue || lastStateValue < curStateValue) {
     629      //    stateChangeInStep = true;
     630      //    changeDetectionCount++;
     631      //  }
     632      //}
     633      //lastStateValue = curStateValue;
     634
     635      // detection accuracy
     636      //if (!stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     637      //  // (stable) state - detected
     638      //  accuracySum++;
     639      //} else if(!stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     640      //  // (stable) state - not detected
     641      //  // no accuracy points (or penalty)
     642      //} else if (stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     643      //  // change - detected
     644      //  accuracySum++;
     645      //} else if (stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     646      //  // change - not detected
     647      //  // no accuracy points (or penalty)
     648      //}
     649    }
     650    private bool accDiscount = false;
     651
     652    private double CalulateEnsembleConfidenceCM1(RatedEnsembleModel ensemble, IEnumerable<double[]> estimatedValuesPerModelPerRow, IEnumerable<double> realValues) {
     653      int noModels = estimatedValuesPerModelPerRow.Count();
     654      int votes = 0;
     655
     656      OnlineCalculatorError error;
     657
     658      foreach(var model in estimatedValuesPerModelPerRow) {
     659        var pR = OnlinePearsonsRCalculator.Calculate(model, realValues, out error);
     660        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
     661        if (pR2 >= ensemble.QualityThreshold.End) {
     662          votes++;
     663        }
     664      }
     665     
     666
     667      return (double)votes/noModels;
    450668    }
    451669
    … …  
    499717
    500718    protected virtual void RegisterEnsembleEvents() {
    501       ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    502       ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    503       ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    504       ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    505       ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_Reset);
     719      ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     720      ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     721      ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     722      ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     723      ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    506724    }
    507725
    508726    protected virtual void DeregisterEnsembleEvents() {
    509       ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    510       ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    511       ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    512       ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    513       ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_Reset);
     727      ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     728      ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     729      ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     730      ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     731      ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    514732    }
    515733    #endregion

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/HeuristicLab.DatastreamAnalysis.csproj

    r14588 r14710  
    110110    <Compile Include="IAnalysisBase.cs" />
    111111    <Compile Include="IDatastream.cs" />
     112    <Compile Include="IRatedEnsembleModel.cs" />
    112113    <Compile Include="Plugin.cs" />
    113114    <Compile Include="Properties\AssemblyInfo.cs" />
     115    <Compile Include="RatedEnsembleModel.cs" />
    114116  </ItemGroup>
    115117  <ItemGroup>

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Plugin.cs

    r14588 r14710  
    2323
    2424namespace HeuristicLab.DatastreamAnalysis {
    25   [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14547")]
     25  [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14588")]
    2626  [PluginFile("HeuristicLab.DatastreamAnalysis-3.4.dll", PluginFileType.Assembly)]
    2727  [PluginDependency("HeuristicLab.Collections", "3.3")]

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Properties/AssemblyInfo.cs

    r14588 r14710  
    5454// [assembly: AssemblyVersion("1.0.*")]
    5555[assembly: AssemblyVersion("3.4.0.0")]
    56 [assembly: AssemblyFileVersion("3.4.14.14547")]
     56[assembly: AssemblyFileVersion("3.4.14.14588")]
Note: See TracChangeset for help on using the changeset viewer.