Free cookie consent management tool by TermsFeed Policy Generator

Changeset 14710


Ignore:
Timestamp:
03/02/17 14:10:57 (8 years ago)
Author:
jzenisek
Message:

#2719 implemented ensemble model rating by introducing the new type RatedEnsembleModel; introduced performance indicator calculation in results;

Location:
branches/HeuristicLab.DatastreamAnalysis
Files:
4 added
13 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/DataBarSetView.cs

    r14588 r14710  
    119119
    120120        chart.Series.Clear();
     121       
    121122        Series series = chart.Series.Add(seriesName);
    122123        series.IsVisibleInLegend = false;
    … …  
    124125        int i = 0;
    125126        foreach (var bar in Content.Bars) {
    126           series.Points.Add(new DataPoint(i, bar.Value) { AxisLabel = bar.Name, Color = bar.BarColor });
     127          var dp = new DataPoint(i, bar.Value) {
     128            AxisLabel = bar.Name,
     129            ToolTip = bar.Name,
     130            Color = bar.BarColor,
     131            BorderWidth = 5
     132          };
     133          series.Points.Add(dp);
    127134          i++;
    128135        }
    … …  
    154161        for (int i = 0; i < bars.Count; i++) {
    155162          chart.Series[seriesName].Points.ElementAt(i).SetValueY(bars[i].Value);
     163          chart.Series[seriesName].Points.ElementAt(i).ToolTip = bars[i].Name + " = " + bars[i].Value;
    156164        }
    157165
    … …  
    163171    }
    164172
     173    public static Color DefaultColor = Color.FromArgb(65, 140, 240);
     174    public static Color HighlightColor1 = Color.FromArgb(37, 105, 175);
     175    public static Color HighlightColor2 = Color.FromArgb(247, 150, 70);
     176    public static Color HighlightColor3 = Color.FromArgb(228, 108, 10);
     177    //public static Color HighlightColor1 = Color.FromArgb(45, 120, 200);
     178    //public static Color HighlightColor2 = Color.FromArgb(15, 90, 160);
     179    //public static Color HighlightColor3 = Color.FromArgb(10, 60, 110);
    165180    private void CheckThresholds() {
    166181      var bars = Content.Bars.ToList();
     182      var winner =
     183        !bars.Any() ? -1 :
     184        bars
     185        .Select((value, index) => new {Value = value, Index = index})
     186        .Aggregate((a, b) => ((a.Value.Value - a.Value.ThresholdUpperBound) > (b.Value.Value - b.Value.ThresholdUpperBound)) ? a : b)
     187        .Index;
     188
     189      // color bars depending on the threshold boundaries they exceed
    167190      for (int i = 0; i < bars.Count; i++) {
    168191        var bar = bars[i];
    169         if (bar.Value > bar.Threshold) {
    170           chart.Series[seriesName].Points[i].Color = Color.Orange;
     192        if (bar.Value > bar.ThresholdUpperBound) {
     193          chart.Series[seriesName].Points[i].Color = HighlightColor2;
     194          chart.Series[seriesName].Points[i].BorderColor = HighlightColor2;
     195        } else if (bar.Value > bar.ThresholdLowerBound) {
     196          chart.Series[seriesName].Points[i].Color = HighlightColor1;
     197          chart.Series[seriesName].Points[i].BorderColor = HighlightColor1;
    171198        } else {
    172199          chart.Series[seriesName].Points[i].Color = bar.BarColor;
     200          chart.Series[seriesName].Points[i].BorderColor = bar.BarColor;
    173201        }
    174202      }
    175     }
    176 
     203
     204      // color the winner's border
     205      if (winner != -1 && bars[winner].Value > bars[winner].ThresholdUpperBound) {
     206        chart.Series[seriesName].Points[winner].BorderColor = HighlightColor3;
     207      }
     208    }
     209
     210
     211
     212    private void AddThresholds() {
     213      Series series = chart.Series.Add("Thresholds");
     214      series.IsVisibleInLegend = false;
     215      series.ChartType = SeriesChartType.ErrorBar;
     216
     217      int i = 0;
     218      foreach (var bar in Content.Bars) {
     219        string desc = string.Format("{0} - Threshold [{1},{2}]", bar.Name, bar.ThresholdLowerBound,bar.ThresholdUpperBound);
     220        var threshold = new DataPoint(i, new[]
     221              {(bar.ThresholdLowerBound + bar.ThresholdUpperBound)/2, bar.ThresholdLowerBound, bar.ThresholdUpperBound}) {
     222                AxisLabel = desc,
     223                ToolTip = desc,
     224                Color = bar.ThresholdColor,
     225                BorderWidth = 1
     226        };
     227        series.Points.Add(threshold);
     228        i++;
     229      }
     230    }
    177231    private void AddThresholdsOld() {
    178232      Series series = chart.Series.Add("Thresholds");
    … …  
    183237      foreach (var bar in Content.Bars) {
    184238        //series.Points.Add(new DataPoint(i, new [] { bar.Threshold, bar.Threshold + 0.05 }) { AxisLabel = bar.Name, Color = Color.Transparent, BorderWidth = 2, BorderColor = bar.ThresholdColor});
    185         series.Points.Add(new DataPoint(i, new[] { bar.Threshold, bar.Threshold + 0.05 }) { AxisLabel = bar.Name, Color = bar.ThresholdColor });
     239        series.Points.Add(new DataPoint(i, new[] { bar.ThresholdLowerBound, bar.ThresholdUpperBound }) { AxisLabel = bar.Name, Color = bar.ThresholdColor });
    186240        i++;
    187241      }
    188242    }
    189 
    190     private void AddThresholds() {
     243    private void AddThresholdsOld2() {
    191244      chart.Annotations.Clear();
    192245
    … …  
    202255        annotation.AxisX = chart.ChartAreas[0].AxisX;
    203256        annotation.AxisY = chart.ChartAreas[0].AxisY;
    204         annotation.Y = bars[i].Threshold;
     257        annotation.Y = bars[i].ThresholdLowerBound;
    205258
    206259        annotation.Alignment = ContentAlignment.MiddleCenter;
    … …  
    208261
    209262        annotation.X = i - 0.4;
    210         annotation.Width = 52.0 / bars.Count;
     263        annotation.Width = (46 - bars.Count) / bars.Count;
    211264        annotation.Name = bars[i].Name;
    212265        annotation.Tag = bars[i].Name;

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/DatastreamAnalysisOptimizerView.cs

    r14538 r14710  
    157157        try {
    158158          var data = e.Data.GetData(HeuristicLab.Common.Constants.DragDropDataFormat);
    159           if (data is IRegressionEnsembleModel) {
    160             data = (IRegressionEnsembleModel) data;
    161             Content.Ensembles.Add((RegressionEnsembleModel) data);
     159          if (data is IRatedEnsembleModel) {
     160            data = (IRatedEnsembleModel) data;
     161            Content.Ensembles.Add((RatedEnsembleModel) data);
    162162          }
    163163        }
    … …  
    175175      var data = e.Data.GetData(HeuristicLab.Common.Constants.DragDropDataFormat);
    176176
    177       if (data is IRegressionEnsembleModel) {
     177      if (data is IRatedEnsembleModel) {
    178178        if ((e.KeyState & 32) == 32) e.Effect = DragDropEffects.Link; // ALT key
    179179        else if ((e.KeyState & 4) == 4) e.Effect = DragDropEffects.Move; // SHIFT key

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/HeuristicLab.DatastreamAnalysis.Views.csproj

    r14588 r14710  
    111111      <DependentUpon>DatastreamAnalysisOptimizerView.cs</DependentUpon>
    112112    </Compile>
     113    <Compile Include="Form1.cs">
     114      <SubType>Form</SubType>
     115    </Compile>
     116    <Compile Include="Form1.Designer.cs">
     117      <DependentUpon>Form1.cs</DependentUpon>
     118    </Compile>
    113119    <Compile Include="Plugin.cs" />
    114120    <Compile Include="Properties\AssemblyInfo.cs" />
    … …  
    142148      <DependentUpon>DatastreamAnalysisOptimizerView.cs</DependentUpon>
    143149    </EmbeddedResource>
     150    <EmbeddedResource Include="Form1.resx">
     151      <DependentUpon>Form1.cs</DependentUpon>
     152    </EmbeddedResource>
    144153    <EmbeddedResource Include="RegressionEnsembleModelView.resx">
    145154      <DependentUpon>RegressionEnsembleModelView.cs</DependentUpon>

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/Plugin.cs

    r14588 r14710  
    2323
    2424namespace HeuristicLab.DatastreamAnalysis.Views {
    25   [Plugin("HeuristicLab.DatastreamAnalysis.Views", "3.4.14.14547")]
     25  [Plugin("HeuristicLab.DatastreamAnalysis.Views", "3.4.14.14588")]
    2626  [PluginFile("HeuristicLab.DatastreamAnalysis.Views-3.4.dll", PluginFileType.Assembly)]
    2727  [PluginDependency("HeuristicLab.Collections", "3.3")]

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis.Views/3.4/Properties/AssemblyInfo.cs

    r14588 r14710  
    5454// [assembly: AssemblyVersion("1.0.*")]
    5555[assembly: AssemblyVersion("3.4.0.0")]
    56 [assembly: AssemblyFileVersion("3.4.14.14547")]
     56[assembly: AssemblyFileVersion("3.4.14.14588")]

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DataBar.cs

    r14588 r14710  
    2828using HeuristicLab.Common;
    2929using HeuristicLab.Core;
     30using HeuristicLab.Data;
    3031using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    3132
    … …  
    3839  public class DataBar : NamedItem, INotifyPropertyChanged {
    3940    private double value;
     41    public static Color DefaultBarColor = Color.FromArgb(65, 140, 240);
    4042
    4143    public double Value {
    … …  
    5052    }
    5153
    52     private double threshold;
     54    private double thresholdLowerBound;
    5355
    54     public double Threshold {
    55       get { return threshold; }
     56    public double ThresholdLowerBound {
     57      get { return thresholdLowerBound; }
    5658      set {
     59        if (this.value != value) {
     60          this.value = value;
     61          OnThresholdPropertyChanged("Threshold");
     62        }
     63      }
     64    }
     65
     66    private double thresholdUpperBound;
     67
     68    public double ThresholdUpperBound
     69    {
     70      get { return thresholdUpperBound; }
     71      set
     72      {
    5773        if (this.value != value) {
    5874          this.value = value;
    … …  
    6884      set { this.value = value; }
    6985    }
    70     [Storable(Name = "Threshold")]
    71     private double StorableThreshold {
    72       get { return threshold; }
    73       set { this.threshold = value; }
     86    [Storable(Name = "ThresholdLowerBound")]
     87    private double StorableThresholdLowerBound {
     88      get { return thresholdLowerBound; }
     89      set { this.thresholdLowerBound = value; }
     90    }
     91    [Storable(Name = "ThresholdUpperBound")]
     92    private double StorableThresholdUpperBound
     93    {
     94      get { return thresholdUpperBound; }
     95      set { this.thresholdUpperBound = value; }
    7496    }
    7597    #endregion
    … …  
    109131    protected DataBar(DataBar original, Cloner cloner) : base(original, cloner) {
    110132      this.value = original.value;
    111       this.threshold = original.threshold;
     133      this.thresholdLowerBound = original.thresholdLowerBound;
    112134      this.barColor = original.barColor;
    113135      this.thresholdColor = original.thresholdColor;
    … …  
    115137
    116138    public DataBar() : this("DataBar") {
    117       barColor = Color.Empty;
    118       thresholdColor = Color.Red;
     139      barColor = DefaultBarColor;
     140      thresholdColor = Color.Black;
    119141    }
    120142
    121143    public DataBar(string name) : base(name) {
    122144      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    123       barColor = Color.Empty;
    124       thresholdColor = Color.Red;
     145      barColor = DefaultBarColor;
     146      thresholdColor = Color.Black;
    125147    }
    126148
    127     public DataBar(string name, double threshold) : base(name) {
     149    public DataBar(string name, double thresholdLowerBound, double thresholdUpperBound) : base(name) {
    128150      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    129       this.threshold = threshold;
    130       barColor = Color.Empty;
    131       thresholdColor = Color.Red;
     151      this.thresholdLowerBound = thresholdLowerBound;
     152      this.thresholdUpperBound = thresholdUpperBound;
     153      barColor = DefaultBarColor;
     154      thresholdColor = Color.Black;
    132155    }
    133156
    134157    public DataBar(string name, string description) : base(name, description) {
    135158      if (string.IsNullOrEmpty(name)) throw new ArgumentException("Name of a DataBar cannot be empty", name);
    136       barColor = Color.Empty;
    137       thresholdColor = Color.Red;
     159      barColor = DefaultBarColor;
     160      thresholdColor = Color.Black;
    138161    }
    139162

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Datastream.cs

    r14543 r14710  
    4242    protected const string SlidingWindowSizeParameterName = "SlidingWindowSize";
    4343    protected const string SlidingWindowStepWidthParameterName = "SlidingWindowStepWidth";
    44     protected const string SlidingWindowMovementIntervalParameterName = "SlidingWindowMovementInterval";
     44    protected const string SlidingWindowMovementDelayParameterName = "SlidingWindowMovementDelay";
    4545    protected const string FitnessPartitionParameterName = "FitnessPartition";
     46    protected const string SlidingWindowSonarRatioParameterName = "SlidingWindowSonarRatio";
    4647
    4748    #region parameter properites
    … …  
    6364    }
    6465
    65     public IValueParameter<IntValue> SlidingWindowMovementIntervalParameter {
    66       get { return (IValueParameter<IntValue>) Parameters[SlidingWindowMovementIntervalParameterName]; }
     66    public IValueParameter<IntValue> SlidingWindowMovementDelayParameter {
     67      get { return (IValueParameter<IntValue>) Parameters[SlidingWindowMovementDelayParameterName]; }
     68    }
     69
     70    public IValueParameter<DoubleValue> SlidingWindowSonarRatioParameter {
     71      get { return (IValueParameter<DoubleValue>) Parameters[SlidingWindowSonarRatioParameterName]; }
    6772    }
    6873
    … …  
    7883      get { return ProblemDataParameter.Value; }
    7984      set {
    80         if(value == null) throw new ArgumentNullException("problemData", "The provided value for the problemData is null.");
     85        if(value == null) throw new ArgumentNullException("ProblemData", "The provided value for problemData is null.");
    8186        ProblemDataParameter.Value = value;
    8287        //OnProblemDataChanged();
    … …  
    100105    }
    101106
    102     public IntValue SlidingWindowMovementInterval {
    103       get { return SlidingWindowMovementIntervalParameter.Value; }
    104       set { SlidingWindowMovementIntervalParameter.Value = value; }
     107    public IntValue SlidingWindowMovementDelay {
     108      get { return SlidingWindowMovementDelayParameter.Value; }
     109      set { SlidingWindowMovementDelayParameter.Value = value; }
     110    }
     111
     112    public DoubleValue SlidingWindowSonarRatio {
     113      get { return SlidingWindowSonarRatioParameter.Value; }
     114      set { SlidingWindowSonarRatioParameter.Value = value; }
    105115    }
    106116
    … …  
    181191    public Datastream() : base() {
    182192      Parameters.Add(new ValueParameter<RegressionProblemData>(ProblemDataParameterName, "ProblemData for analysis with selected ensembles.", null));
    183       Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 20)));
    184       Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(20)));
     193      Parameters.Add(new FixedValueParameter<IntRange>(InitialSlidingWindowParameterName, "Initial sliding window boundaries", new IntRange(0, 100)));
     194      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowSizeParameterName, "Sliding window size", new IntValue(100)));
    185195      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowStepWidthParameterName, "Sliding window step width", new IntValue(1)));
    186       Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementIntervalParameterName, "Sliding window movement interval (milliseconds)", new IntValue(500)));
     196      Parameters.Add(new FixedValueParameter<IntValue>(SlidingWindowMovementDelayParameterName, "Sliding window movement delay interval (milliseconds)", new IntValue(0)));
     197      Parameters.Add(new FixedValueParameter<DoubleValue>(SlidingWindowSonarRatioParameterName, "Sliding window sonar ratio", new DoubleValue(0.25)));
    187198      RegisterParameterEvents();
    188199      InitializeState();

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/DatastreamAnalysisOptimizer.cs

    r14588 r14710  
    7474    }
    7575
     76    private double lastStateValue;
     77    private double pR2Sum;
     78    private int detectionCount;
     79    private int evaluationCount;
     80    private double exclusivitySum;
     81    private double accuracySum;
     82
    7683    private CancellationTokenSource cancellationTokenSource;
    7784    private bool stopPending;
    … …  
    100107    }
    101108
    102 
    103109    [Storable]
    104     private IItemList<RegressionEnsembleModel> ensembles;
    105 
    106     public IItemList<RegressionEnsembleModel> Ensembles {
     110    private IItemList<RatedEnsembleModel> ensembles;
     111
     112    public IItemList<RatedEnsembleModel> Ensembles {
    107113      get { return ensembles; }
    108114      set {
    109115        if (value == null || value == ensembles)
    110116          return;
    111         if(!(value is IRegressionEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
     117        if (!(value is IRatedEnsembleModel)) throw new ArgumentException("Invaid ensemble model type");
    112118        DeregisterEnsembleEvents();
    113119        ensembles = value;
    … …  
    145151    }
    146152    private DataTable ResultsQualities {
    147       get { return ((DataTable)Results["Qualities"].Value); }
     153      get { return ((DataTable)Results["Qualities 1"].Value); }
     154    }
     155
     156    private DataTable ResultsPeakQualities {
     157      get { return ((DataTable) Results["Qualities 2"].Value); }
    148158    }
    149159
    … …  
    155165    }
    156166
     167    private DataTable ResultsStateDetection {
     168      get { return ((DataTable) Results["State Detection"].Value);}
     169    }
     170
    157171    private const string ResultsTargetsReal = "Real";
    158172
    159     private DataBarSet ResultsQualitiesBars {
    160       get { return (DataBarSet) Results["Ensemble Comparison"].Value; }
    161     }
     173    private DataBarSet ResultsSWQualitiesBars {
     174      get { return (DataBarSet) Results["Ensemble Comparison [SW-Quality]"].Value; }
     175    }
     176
     177    private DataBarSet ResultsSWVotingBars {
     178      get { return (DataBarSet) Results["Ensemble Comparison [SW-Voting]"].Value; }
     179    }
     180
     181    private DataBarSet ResultsSonarQualitiesBars {
     182      get { return (DataBarSet)Results["Ensemble Comparison [Sonar-Quality]"].Value; }
     183    }
     184
     185    private DataBarSet ResultsSWPeakQualitiesBars {
     186      get { return (DataBarSet)Results["Ensemble Comparison [SW-PeakQuality]"].Value; }
     187    }
     188
     189    private double ResultsMeanPR2 {
     190      get { return ((DoubleValue) Results["Mean PR2"].Value).Value; }
     191      set { ((DoubleValue)Results["Mean PR2"].Value).Value = value; }
     192    }
     193
     194    private double ResultsDetectionAccuracy {
     195      get { return ((DoubleValue)Results["Detection Accuracy"].Value).Value; }
     196      set { ((DoubleValue)Results["Detection Accuracy"].Value).Value = value; }
     197    }
     198
     199    private double ResultsDetectionExclusivity {
     200      get { return ((DoubleValue)Results["Detection Exclusivity"].Value).Value; }
     201      set { ((DoubleValue)Results["Detection Exclusivity"].Value).Value = value; }
     202    }
     203
     204    //private int ResultsTrueDetectionCount {
     205    //  get { return ((IntValue)Results["True Detection Count"].Value).Value; }
     206    //  set { ((IntValue)Results["True Detection Count"].Value).Value = value; }
     207    //}
     208
     209    //private int ResultsFalseDetectionCount {
     210    //  get { return ((IntValue)Results["False Detection Count"].Value).Value; }
     211    //  set { ((IntValue)Results["False Detection Count"].Value).Value = value; }
     212    //}
    162213
    163214    protected void SetupResults() {
     215      evaluationCount = 0;
     216      lastStateValue = 0.0;
     217      pR2Sum = 0.0;
     218      exclusivitySum = 0.0;
     219      accuracySum = 0.0;
     220      detectionCount = 0;
    164221      Results.Clear();
    165 
    166222      Results.Add(new Result("Sliding Window Movements", new IntValue(0)));
    167       Results.Add(new Result("Qualities", new DataTable("Pearson R²")));
     223      Results.Add(new Result("Qualities 1", new DataTable("Average Pearson R²")));
     224      Results.Add(new Result("Qualities 2", new DataTable("Peak Pearson R²")));
    168225      Results.Add(new Result("Targets", new DataTable("Targets")));
    169       Results.Add(new Result("Ensemble Comparison", new DataBarSet("Ensemble Comparison")));
     226      Results.Add(new Result("State Detection", new DataTable("State Detection")));
     227      Results.Add(new Result("Ensemble Comparison [SW-Quality]", new DataBarSet("Ensemble Comparison [SW-Quality]")));
     228      Results.Add(new Result("Ensemble Comparison [SW-Voting]", new DataBarSet("Ensemble Comparison [SW-Voting]")));
     229      Results.Add(new Result("Ensemble Comparison [Sonar-Quality]", new DataBarSet("Ensemble Comparison [Sonar-Quality]")));
     230      Results.Add(new Result("Ensemble Comparison [SW-PeakQuality]", new DataBarSet("Ensemble Comparison [SW-PeakQuality]")));
     231      Results.Add(new Result("Mean PR2", new DoubleValue()));
     232      Results.Add(new Result("Detection Accuracy", new DoubleValue()));
     233      Results.Add(new Result("Detection Exclusivity", new DoubleValue()));
    170234
    171235      ResultsTargets.Rows.Add(new DataRow(ResultsTargetsReal));
    … …  
    174238        ResultsTargets.Rows.Add(new DataRow(ensemble.Name));
    175239
     240        ResultsStateDetection.Rows.Add(new DataRow(ensemble.Name));
     241
    176242        // qualities (series)
    177243        //ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesMSE));
    178         ResultsQualities.Rows.Add(new DataRow(ensemble.Name + " - " + ResultsQualitiesPR2));
     244        ResultsQualities.Rows.Add(new DataRow(ensemble.Name));
     245
     246        ResultsPeakQualities.Rows.Add(new DataRow(ensemble.Name));
    179247
    180248        // qualities (bars)
    181         ResultsQualitiesBars.Bars.Add(new DataBar(ensemble.Name, 0.9));
     249        ResultsSWQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
     250
     251        // voting (bars)
     252        ResultsSWVotingBars.Bars.Add(new DataBar(ensemble.Name, ensemble.ConfidenceThreshold.Start, ensemble.ConfidenceThreshold.End));
     253
     254        // sonar quality (bars)
     255        ResultsSonarQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
     256
     257        // quality peaks (bars)
     258        ResultsSWPeakQualitiesBars.Bars.Add(new DataBar(ensemble.Name, ensemble.QualityThreshold.Start, ensemble.QualityThreshold.End));
    182259      }
    183260    }
    … …  
    190267      log = new Log();
    191268      results = new ResultCollection();
    192       ensembles = new ItemList<RegressionEnsembleModel>();
     269      ensembles = new ItemList<RatedEnsembleModel>();
    193270      datastream = new Datastream();
    194271      runsCounter = 0;
    … …  
    210287      log = cloner.Clone(original.log);
    211288      results = cloner.Clone(original.results);
    212       ensembles = (ItemList<RegressionEnsembleModel>) original.Ensembles.Clone(cloner);
     289      ensembles = (ItemList<RatedEnsembleModel>) original.Ensembles.Clone(cloner);
    213290      datastream = (Datastream) original.Datastream.Clone(cloner);
    214291      runsCounter = original.runsCounter;
    … …  
    373450        // play and evaluate initial window
    374451        PlayDatastream();
    375         if(Datastream.SlidingWindowEvaluationPossible) Evaluate();
     452        lastStateValue = Datastream.ProblemData.Dataset.GetDoubleValue("h", replayedIndex);
     453        if (Datastream.SlidingWindowEvaluationPossible) Evaluate();
    376454        replayedIndex = Datastream.FitnessPartition.End;
    377455
    … …  
    381459
    382460            // perform (delayed) window movement
    383             Thread.Sleep(Datastream.SlidingWindowMovementInterval.Value);
     461            Thread.Sleep(Datastream.SlidingWindowMovementDelay.Value);
    384462            Datastream.MoveSlidingWindow();
    385463            ResultsSlidingWindowMovements++;
    … …  
    412490      }
    413491    }
    414 
     492   
    415493    private void Evaluate() {
     494      evaluationCount++;
    416495      var problemData = Datastream.ProblemData;
    417496      var dataset = problemData.Dataset;
    418497      var targetVarName = problemData.TargetVariable;
    419498
    420       var realRows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size).ToList();
    421       var realValues = dataset.GetDoubleValues(targetVarName, realRows);
    422 
     499      var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
     500      var realValues = dataset.GetDoubleValues(targetVarName, rows);
     501
     502      int sonarSize = (int)(Datastream.FitnessPartition.Size * Datastream.SlidingWindowSonarRatio.Value);
     503      var sonarStart = Datastream.FitnessPartition.End - sonarSize;
     504      var sonarRows = Enumerable.Range(sonarStart, sonarSize);
     505      var sonarRealValues = dataset.GetDoubleValues(targetVarName, sonarRows);
     506
     507      var winningEnsembleName = ensembles[0].Name;
     508      var winningEnsemblePR2 = 0.0;
     509      var winningEnsembleIdx = 0;
     510      var ec = 0;
     511      bool stateChangeInStep = false;
     512      var curDetectionCount = 0;
     513      var changeDetectionCount = 0;
     514      double curStateValue = 0.0;
    423515      foreach (var ensemble in Ensembles) {
    424516
    425         var rows = Enumerable.Range(Datastream.FitnessPartition.Start, Datastream.FitnessPartition.Size);
    426         var estimatedValuesPerModelPerRow = ensemble.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
    427         var estimatedValues = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
     517        var sonarEstimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, sonarRows).ToArray());
     518        var sonarEstimatedValuesPerRow = Enumerable.Range(0, sonarSize).Select(r => sonarEstimatedValuesPerModelPerRow.Select(m => m[r]).Average()).ToArray();
     519
     520        var estimatedValuesPerModelPerRow = ensemble.Model.Models.Select(x => x.GetEstimatedValues(datastream.ProblemData.Dataset, rows).ToArray());
     521        var estimatedValuesPerRow = Enumerable.Range(0, Datastream.FitnessPartition.Size).Select(r => estimatedValuesPerModelPerRow.Select(e => e[r]).Average()).ToArray(); // per row
    428522        var averageEstimatedValuesPerModel = estimatedValuesPerModelPerRow.Select(x => x.Average()); // per model
    429523        var averageEstimatedValue = averageEstimatedValuesPerModel.Average();
    430524
    431         // determine quality
    432         var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
     525        // calulate quality
     526        //var mse = Math.Pow(averageEstimatedValue - realValues.Average(), 2);
    433527        OnlineCalculatorError error;
    434         var pR = OnlinePearsonsRCalculator.Calculate(estimatedValues, realValues, out error);
     528        var pR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerRow, realValues, out error);
    435529        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
    436530
     531        var sonarPR = OnlinePearsonsRCalculator.Calculate(sonarEstimatedValuesPerRow, sonarRealValues, out error);
     532        var sonarPR2 = error == OnlineCalculatorError.None ? sonarPR * sonarPR : 0.0;
     533   
     534        var estimatedValuesPerModelPerRowArr = estimatedValuesPerModelPerRow.ToArray();
     535        var curPR = 0.0;
     536       
     537        var peakPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[0], realValues, out error);
     538        for (int i = 1; i < estimatedValuesPerModelPerRow.Count(); i++) {
     539          curPR = OnlinePearsonsRCalculator.Calculate(estimatedValuesPerModelPerRowArr[i], realValues, out error);
     540          if (curPR > peakPR && error == OnlineCalculatorError.None) {
     541            peakPR = curPR;
     542          }
     543        }
     544        var peakPR2 = peakPR * peakPR;
     545
     546        // calculate confidence
     547        var cm1 = CalulateEnsembleConfidenceCM1(ensemble, estimatedValuesPerModelPerRow, realValues);
     548
    437549        int replayAmount = Datastream.FitnessPartition.End - replayedIndex;
    438         int replayCount = estimatedValues.Length - replayAmount;
     550        int replayCount = estimatedValuesPerRow.Length - replayAmount;
    439551
    440552        for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
    441553          //ResultsTargets.Rows[ensemble.Name].Values.Add(averageEstimatedValue);
    442           ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValues[replayCount]);
     554          ResultsTargets.Rows[ensemble.Name].Values.Add(estimatedValuesPerRow[replayCount]);
    443555          replayCount++;
    444556
    445557          //ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesMSE].Values.Add(mse);
    446           ResultsQualities.Rows[ensemble.Name + " - " + ResultsQualitiesPR2].Values.Add(pR2);
    447           ResultsQualitiesBars.Bars[ensemble.Name].Value = pR2;
    448         }
    449       }
     558          ResultsQualities.Rows[ensemble.Name].Values.Add(pR2);
     559          ResultsPeakQualities.Rows[ensemble.Name].Values.Add(peakPR2);
     560
     561          double amp = (pR2 > ensemble.QualityThreshold.Start) ? 1.0 : 0.0;
     562          ResultsStateDetection.Rows[ensemble.Name].Values.Add(amp);   
     563        }
     564        ResultsSWQualitiesBars.Bars[ensemble.Name].Value = pR2;
     565        ResultsSonarQualitiesBars.Bars[ensemble.Name].Value = sonarPR2;
     566        ResultsSWVotingBars.Bars[ensemble.Name].Value = cm1;
     567        ResultsSWPeakQualitiesBars.Bars[ensemble.Name].Value = peakPR2;
     568
     569        if (pR2 > ensemble.QualityThreshold.Start)
     570          curDetectionCount++;
     571
     572        if (pR2 > winningEnsemblePR2) {
     573          winningEnsemblePR2 = pR2;
     574          winningEnsembleName = ensemble.Name;
     575          winningEnsembleIdx = ec;
     576        }
     577        ec++;
     578      }
     579
     580      // quality meassures
     581      // =================================================
     582
     583      if (winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     584        // Mean PR2
     585        pR2Sum += winningEnsemblePR2;
     586        detectionCount++;
     587        ResultsMeanPR2 = pR2Sum/detectionCount;
     588
     589        // detection exclusivity --> 1.0 - (x - 1) * (1.0 / n)
     590        var fraction = 1.0 / ensembles.Count;
     591        var curExclusivity = 1.0 - (curDetectionCount - 1) * fraction;
     592        exclusivitySum += curExclusivity;
     593        ResultsDetectionExclusivity = exclusivitySum / detectionCount;
     594      }
     595
     596      double replCount = 0.0;
     597      double accCount = 0.0;
     598      for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
     599        replCount++;
     600        curStateValue = problemData.Dataset.GetDoubleValue("h", i);
     601        if (curStateValue <= 0.5 && winningEnsembleIdx == 0 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     602          accCount++;
     603        } else if (curStateValue > 0.5 && winningEnsembleIdx == 1 && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     604          accCount++;
     605        }
     606      }
     607      accuracySum += accCount / replCount;
     608      ResultsDetectionAccuracy = accuracySum / evaluationCount;
     609
     610      //if (accCount / replCount > 1.0) {
     611      //  Console.WriteLine();
     612      //}
     613
     614      //if(accuracySum / evaluationCount > 1.0) {
     615      //  Console.WriteLine();
     616      //}
     617
     618      //if (accCount < replCount && !accDiscount) accDiscount = true;
     619      //if (accDiscount && accuracySum/evaluationCount >= 1.0) {
     620      //  Console.WriteLine();
     621      //}
     622
     623      lastStateValue = curStateValue;
     624
     625      // detection of state change
     626      //for (int i = replayedIndex; i < Datastream.FitnessPartition.End; i++) {
     627      //  curStateValue = problemData.Dataset.GetDoubleValue("h", i);
     628      //  if (lastStateValue > curStateValue || lastStateValue < curStateValue) {
     629      //    stateChangeInStep = true;
     630      //    changeDetectionCount++;
     631      //  }
     632      //}
     633      //lastStateValue = curStateValue;
     634
     635      // detection accuracy
     636      //if (!stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     637      //  // (stable) state - detected
     638      //  accuracySum++;
     639      //} else if(!stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     640      //  // (stable) state - not detected
     641      //  // no accuracy points (or penalty)
     642      //} else if (stateChangeInStep && winningEnsemblePR2 < ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     643      //  // change - detected
     644      //  accuracySum++;
     645      //} else if (stateChangeInStep && winningEnsemblePR2 > ensembles[winningEnsembleIdx].QualityThreshold.Start) {
     646      //  // change - not detected
     647      //  // no accuracy points (or penalty)
     648      //}
     649    }
     650    private bool accDiscount = false;
     651
     652    private double CalulateEnsembleConfidenceCM1(RatedEnsembleModel ensemble, IEnumerable<double[]> estimatedValuesPerModelPerRow, IEnumerable<double> realValues) {
     653      int noModels = estimatedValuesPerModelPerRow.Count();
     654      int votes = 0;
     655
     656      OnlineCalculatorError error;
     657
     658      foreach(var model in estimatedValuesPerModelPerRow) {
     659        var pR = OnlinePearsonsRCalculator.Calculate(model, realValues, out error);
     660        var pR2 = error == OnlineCalculatorError.None ? pR * pR : 0.0;
     661        if (pR2 >= ensemble.QualityThreshold.End) {
     662          votes++;
     663        }
     664      }
     665     
     666
     667      return (double)votes/noModels;
    450668    }
    451669
    … …  
    499717
    500718    protected virtual void RegisterEnsembleEvents() {
    501       ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    502       ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    503       ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    504       ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    505       ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_Reset);
     719      ensembles.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     720      ensembles.ItemsMoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     721      ensembles.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     722      ensembles.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     723      ensembles.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    506724    }
    507725
    508726    protected virtual void DeregisterEnsembleEvents() {
    509       ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    510       ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    511       ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    512       ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_ItemsChanged);
    513       ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<RegressionEnsembleModel>>(Ensembles_Reset);
     727      ensembles.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     728      ensembles.ItemsMoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     729      ensembles.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     730      ensembles.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_ItemsChanged);
     731      ensembles.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<RatedEnsembleModel>>(Ensembles_Reset);
    514732    }
    515733    #endregion

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/HeuristicLab.DatastreamAnalysis.csproj

    r14588 r14710  
    110110    <Compile Include="IAnalysisBase.cs" />
    111111    <Compile Include="IDatastream.cs" />
     112    <Compile Include="IRatedEnsembleModel.cs" />
    112113    <Compile Include="Plugin.cs" />
    113114    <Compile Include="Properties\AssemblyInfo.cs" />
     115    <Compile Include="RatedEnsembleModel.cs" />
    114116  </ItemGroup>
    115117  <ItemGroup>

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Plugin.cs

    r14588 r14710  
    2323
    2424namespace HeuristicLab.DatastreamAnalysis {
    25   [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14547")]
     25  [Plugin("HeuristicLab.DatastreamAnalysis", "3.4.14.14588")]
    2626  [PluginFile("HeuristicLab.DatastreamAnalysis-3.4.dll", PluginFileType.Assembly)]
    2727  [PluginDependency("HeuristicLab.Collections", "3.3")]

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.DatastreamAnalysis/3.4/Properties/AssemblyInfo.cs

    r14588 r14710  
    5454// [assembly: AssemblyVersion("1.0.*")]
    5555[assembly: AssemblyVersion("3.4.0.0")]
    56 [assembly: AssemblyFileVersion("3.4.14.14547")]
     56[assembly: AssemblyFileVersion("3.4.14.14588")]

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

    r14491 r14710  
    173173    <Compile Include="Implementation\Regression\ConstantRegressionModel.cs" />
    174174    <Compile Include="Implementation\Regression\ConstantRegressionSolution.cs" />
     175    <Compile Include="Implementation\Regression\RatedRegressionEnsembleModel.cs" />
    175176    <Compile Include="Implementation\Regression\RegressionEnsembleProblemData.cs" />
    176177    <Compile Include="Implementation\Regression\RegressionEnsembleModel.cs">

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RatedRegressionEnsembleModel.cs

    r14538 r14710  
    2525using HeuristicLab.Common;
    2626using HeuristicLab.Core;
     27using HeuristicLab.Data;
    2728using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    2829
    … …  
    3233  /// </summary>
    3334  [StorableClass]
    34   [Item("Regression Ensemble Model", "A regression model that contains an ensemble of multiple regression models")]
     35  [Item("Rated Regression Ensemble Model", "A regression model that contains an ensemble of multiple regression models")]
    3536  [Creatable(CreatableAttribute.Categories.DataAnalysisEnsembles, Priority = 100)]
    36   public sealed class RegressionEnsembleModel : RegressionModel, IRegressionEnsembleModel {
     37  public sealed class RatedRegressionEnsembleModel : RegressionModel, IRegressionEnsembleModel {
    3738    public override IEnumerable<string> VariablesUsedForPrediction {
    3839      get { return models.SelectMany(x => x.VariablesUsedForPrediction).Distinct().OrderBy(x => x); }
    … …  
    5960      get { return modelWeights; }
    6061      set { modelWeights = value.ToList(); }
     62    }
     63
     64    private DoubleRange qualityThreshold;
     65    public DoubleRange QualityThreshold {
     66      get { return qualityThreshold; }     
     67      set { qualityThreshold = value; }
     68    }
     69    [Storable(Name = "QualityThreshold")]
     70    private DoubleRange StorableQualityThreshold {
     71      get { return qualityThreshold; }
     72      set { qualityThreshold = value; }
     73    }
     74
     75    private DoubleRange confidenceThreshold;
     76    public DoubleRange ConfidenceThreshold
     77    {
     78      get { return confidenceThreshold; }
     79      set { confidenceThreshold = value; }
     80    }
     81    [Storable(Name = "QualityThreshold")]
     82    private DoubleRange StorableConfidenceThreshold
     83    {
     84      get { return confidenceThreshold; }
     85      set { confidenceThreshold = value; }
    6186    }
    6287
    … …  
    90115
    91116    [StorableConstructor]
    92     private RegressionEnsembleModel(bool deserializing) : base(deserializing) { }
    93     private RegressionEnsembleModel(RegressionEnsembleModel original, Cloner cloner)
     117    private RatedRegressionEnsembleModel(bool deserializing) : base(deserializing) { }
     118    private RatedRegressionEnsembleModel(RatedRegressionEnsembleModel original, Cloner cloner)
    94119      : base(original, cloner) {
    95120      this.models = original.Models.Select(cloner.Clone).ToList();
    96121      this.modelWeights = new List<double>(original.ModelWeights);
     122      this.qualityThreshold = cloner.Clone(original.qualityThreshold);
     123      this.confidenceThreshold = cloner.Clone(original.confidenceThreshold);
    97124      this.averageModelEstimates = original.averageModelEstimates;
    98125    }
    99126    public override IDeepCloneable Clone(Cloner cloner) {
    100       return new RegressionEnsembleModel(this, cloner);
    101     }
    102 
    103     public RegressionEnsembleModel() : this(Enumerable.Empty<IRegressionModel>()) { }
    104     public RegressionEnsembleModel(IEnumerable<IRegressionModel> models) : this(models, models.Select(m => 1.0)) { }
    105     public RegressionEnsembleModel(IEnumerable<IRegressionModel> models, IEnumerable<double> modelWeights)
     127      return new RatedRegressionEnsembleModel(this, cloner);
     128    }
     129
     130    public RatedRegressionEnsembleModel() : this(Enumerable.Empty<IRegressionModel>()) { }
     131    public RatedRegressionEnsembleModel(IEnumerable<IRegressionModel> models) : this(models, models.Select(m => 1.0)) { }
     132    public RatedRegressionEnsembleModel(IEnumerable<IRegressionModel> models, IEnumerable<double> modelWeights)
    106133      : base(string.Empty) {
    107134      this.name = ItemName;

  • branches/HeuristicLab.DatastreamAnalysis/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionEnsembleModel.cs

    r14538 r14710  
    3333  [StorableClass]
    3434  [Item("Regression Ensemble Model", "A regression model that contains an ensemble of multiple regression models")]
    35   [Creatable(CreatableAttribute.Categories.DataAnalysisEnsembles, Priority = 100)]
    3635  public sealed class RegressionEnsembleModel : RegressionModel, IRegressionEnsembleModel {
    3736    public override IEnumerable<string> VariablesUsedForPrediction {
Note: See TracChangeset for help on using the changeset viewer.