source: trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs @ 12509

Last change on this file since 12509 was 12509, checked in by mkommend, 7 years ago

#2276: Reintegrated branch for dataset refactoring.

File size: 16.1 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Collections;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.Data;
29using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
30
31namespace HeuristicLab.Problems.DataAnalysis {
32  /// <summary>
33  /// Represents classification solutions that contain an ensemble of multiple classification models
34  /// </summary>
35  [StorableClass]
36  [Item("Classification Ensemble Solution", "A classification solution that contains an ensemble of multiple classification models")]
37  [Creatable(CreatableAttribute.Categories.DataAnalysisEnsembles, Priority = 110)]
38  public sealed class ClassificationEnsembleSolution : ClassificationSolutionBase, IClassificationEnsembleSolution {
39    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
40    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
41    private readonly Dictionary<int, double> evaluationCache = new Dictionary<int, double>();
42
43    public new IClassificationEnsembleModel Model {
44      get { return (IClassificationEnsembleModel)base.Model; }
45    }
46    public new ClassificationEnsembleProblemData ProblemData {
47      get { return (ClassificationEnsembleProblemData)base.ProblemData; }
48      set { base.ProblemData = value; }
49    }
50
51    private readonly ItemCollection<IClassificationSolution> classificationSolutions;
52    public IItemCollection<IClassificationSolution> ClassificationSolutions {
53      get { return classificationSolutions; }
54    }
55
56    [Storable]
57    private Dictionary<IClassificationModel, IntRange> trainingPartitions;
58    [Storable]
59    private Dictionary<IClassificationModel, IntRange> testPartitions;
60
61    [StorableConstructor]
62    private ClassificationEnsembleSolution(bool deserializing)
63      : base(deserializing) {
64      classificationSolutions = new ItemCollection<IClassificationSolution>();
65    }
66    [StorableHook(HookType.AfterDeserialization)]
67    private void AfterDeserialization() {
68      foreach (var model in Model.Models) {
69        IClassificationProblemData problemData = (IClassificationProblemData)ProblemData.Clone();
70        problemData.TrainingPartition.Start = trainingPartitions[model].Start;
71        problemData.TrainingPartition.End = trainingPartitions[model].End;
72        problemData.TestPartition.Start = testPartitions[model].Start;
73        problemData.TestPartition.End = testPartitions[model].End;
74
75        classificationSolutions.Add(model.CreateClassificationSolution(problemData));
76      }
77      RegisterClassificationSolutionsEventHandler();
78    }
79
80    private ClassificationEnsembleSolution(ClassificationEnsembleSolution original, Cloner cloner)
81      : base(original, cloner) {
82      trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
83      testPartitions = new Dictionary<IClassificationModel, IntRange>();
84      foreach (var pair in original.trainingPartitions) {
85        trainingPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
86      }
87      foreach (var pair in original.testPartitions) {
88        testPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
89      }
90
91      trainingEvaluationCache = new Dictionary<int, double>(original.ProblemData.TrainingIndices.Count());
92      testEvaluationCache = new Dictionary<int, double>(original.ProblemData.TestIndices.Count());
93
94      classificationSolutions = cloner.Clone(original.classificationSolutions);
95      RegisterClassificationSolutionsEventHandler();
96    }
97
98    public ClassificationEnsembleSolution()
99      : base(new ClassificationEnsembleModel(), ClassificationEnsembleProblemData.EmptyProblemData) {
100      trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
101      testPartitions = new Dictionary<IClassificationModel, IntRange>();
102      classificationSolutions = new ItemCollection<IClassificationSolution>();
103
104      RegisterClassificationSolutionsEventHandler();
105    }
106
107    public ClassificationEnsembleSolution(IClassificationProblemData problemData) :
108      this(Enumerable.Empty<IClassificationModel>(), problemData) { }
109
110    public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData)
111      : this(models, problemData,
112             models.Select(m => (IntRange)problemData.TrainingPartition.Clone()),
113             models.Select(m => (IntRange)problemData.TestPartition.Clone())
114      ) { }
115
116    public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData, IEnumerable<IntRange> trainingPartitions, IEnumerable<IntRange> testPartitions)
117      : base(new ClassificationEnsembleModel(Enumerable.Empty<IClassificationModel>()), new ClassificationEnsembleProblemData(problemData)) {
118      this.trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
119      this.testPartitions = new Dictionary<IClassificationModel, IntRange>();
120      this.classificationSolutions = new ItemCollection<IClassificationSolution>();
121
122      List<IClassificationSolution> solutions = new List<IClassificationSolution>();
123      var modelEnumerator = models.GetEnumerator();
124      var trainingPartitionEnumerator = trainingPartitions.GetEnumerator();
125      var testPartitionEnumerator = testPartitions.GetEnumerator();
126
127      while (modelEnumerator.MoveNext() & trainingPartitionEnumerator.MoveNext() & testPartitionEnumerator.MoveNext()) {
128        var p = (IClassificationProblemData)problemData.Clone();
129        p.TrainingPartition.Start = trainingPartitionEnumerator.Current.Start;
130        p.TrainingPartition.End = trainingPartitionEnumerator.Current.End;
131        p.TestPartition.Start = testPartitionEnumerator.Current.Start;
132        p.TestPartition.End = testPartitionEnumerator.Current.End;
133
134        solutions.Add(modelEnumerator.Current.CreateClassificationSolution(p));
135      }
136      if (modelEnumerator.MoveNext() | trainingPartitionEnumerator.MoveNext() | testPartitionEnumerator.MoveNext()) {
137        throw new ArgumentException();
138      }
139
140      trainingEvaluationCache = new Dictionary<int, double>(problemData.TrainingIndices.Count());
141      testEvaluationCache = new Dictionary<int, double>(problemData.TestIndices.Count());
142
143      RegisterClassificationSolutionsEventHandler();
144      classificationSolutions.AddRange(solutions);
145    }
146
147    public override IDeepCloneable Clone(Cloner cloner) {
148      return new ClassificationEnsembleSolution(this, cloner);
149    }
150    private void RegisterClassificationSolutionsEventHandler() {
151      classificationSolutions.ItemsAdded += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_ItemsAdded);
152      classificationSolutions.ItemsRemoved += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_ItemsRemoved);
153      classificationSolutions.CollectionReset += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_CollectionReset);
154    }
155
156
157    #region Evaluation
158    public override IEnumerable<double> EstimatedClassValues {
159      get { return GetEstimatedClassValues(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
160    }
161
162    public override IEnumerable<double> EstimatedTrainingClassValues {
163      get {
164        var rows = ProblemData.TrainingIndices;
165        var rowsToEvaluate = rows.Except(trainingEvaluationCache.Keys);
166        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
167        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, (r, m) => RowIsTrainingForModel(r, m) && !RowIsTestForModel(r, m)).GetEnumerator();
168
169        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
170          trainingEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
171        }
172
173        return rows.Select(row => trainingEvaluationCache[row]);
174      }
175    }
176
177    public override IEnumerable<double> EstimatedTestClassValues {
178      get {
179        var rows = ProblemData.TestIndices;
180        var rowsToEvaluate = rows.Except(testEvaluationCache.Keys);
181        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
182        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, RowIsTestForModel).GetEnumerator();
183
184        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
185          testEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
186        }
187
188        return rows.Select(row => testEvaluationCache[row]);
189      }
190    }
191
192    private IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows, Func<int, IClassificationModel, bool> modelSelectionPredicate) {
193      var estimatedValuesEnumerators = (from model in Model.Models
194                                        select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedClassValues(ProblemData.Dataset, rows).GetEnumerator() })
195                                       .ToList();
196      var rowsEnumerator = rows.GetEnumerator();
197      // aggregate to make sure that MoveNext is called for all enumerators
198      while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
199        int currentRow = rowsEnumerator.Current;
200
201        var selectedEnumerators = from pair in estimatedValuesEnumerators
202                                  where modelSelectionPredicate(currentRow, pair.Model)
203                                  select pair.EstimatedValuesEnumerator;
204
205        yield return AggregateEstimatedClassValues(selectedEnumerators.Select(x => x.Current));
206      }
207    }
208
209    private bool RowIsTrainingForModel(int currentRow, IClassificationModel model) {
210      return trainingPartitions == null || !trainingPartitions.ContainsKey(model) ||
211              (trainingPartitions[model].Start <= currentRow && currentRow < trainingPartitions[model].End);
212    }
213
214    private bool RowIsTestForModel(int currentRow, IClassificationModel model) {
215      return testPartitions == null || !testPartitions.ContainsKey(model) ||
216              (testPartitions[model].Start <= currentRow && currentRow < testPartitions[model].End);
217    }
218
219    public override IEnumerable<double> GetEstimatedClassValues(IEnumerable<int> rows) {
220      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
221      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
222      var valuesEnumerator = (from xs in GetEstimatedClassValueVectors(ProblemData.Dataset, rowsToEvaluate)
223                              select AggregateEstimatedClassValues(xs))
224                             .GetEnumerator();
225
226      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
227        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
228      }
229
230      return rows.Select(row => evaluationCache[row]);
231    }
232
233    public IEnumerable<IEnumerable<double>> GetEstimatedClassValueVectors(IDataset dataset, IEnumerable<int> rows) {
234      if (!Model.Models.Any()) yield break;
235      var estimatedValuesEnumerators = (from model in Model.Models
236                                        select model.GetEstimatedClassValues(dataset, rows).GetEnumerator())
237                                       .ToList();
238
239      while (estimatedValuesEnumerators.All(en => en.MoveNext())) {
240        yield return from enumerator in estimatedValuesEnumerators
241                     select enumerator.Current;
242      }
243    }
244
245    private double AggregateEstimatedClassValues(IEnumerable<double> estimatedClassValues) {
246      return estimatedClassValues
247      .GroupBy(x => x)
248      .OrderBy(g => -g.Count())
249      .Select(g => g.Key)
250      .DefaultIfEmpty(double.NaN)
251      .First();
252    }
253    #endregion
254
255    protected override void OnProblemDataChanged() {
256      trainingEvaluationCache.Clear();
257      testEvaluationCache.Clear();
258      evaluationCache.Clear();
259
260      IClassificationProblemData problemData = new ClassificationProblemData(ProblemData.Dataset,
261                                                                     ProblemData.AllowedInputVariables,
262                                                                     ProblemData.TargetVariable);
263      problemData.TrainingPartition.Start = ProblemData.TrainingPartition.Start;
264      problemData.TrainingPartition.End = ProblemData.TrainingPartition.End;
265      problemData.TestPartition.Start = ProblemData.TestPartition.Start;
266      problemData.TestPartition.End = ProblemData.TestPartition.End;
267
268      foreach (var solution in ClassificationSolutions) {
269        if (solution is ClassificationEnsembleSolution)
270          solution.ProblemData = ProblemData;
271        else
272          solution.ProblemData = problemData;
273      }
274      foreach (var trainingPartition in trainingPartitions.Values) {
275        trainingPartition.Start = ProblemData.TrainingPartition.Start;
276        trainingPartition.End = ProblemData.TrainingPartition.End;
277      }
278      foreach (var testPartition in testPartitions.Values) {
279        testPartition.Start = ProblemData.TestPartition.Start;
280        testPartition.End = ProblemData.TestPartition.End;
281      }
282
283      base.OnProblemDataChanged();
284    }
285
286    public void AddClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
287      classificationSolutions.AddRange(solutions);
288
289      trainingEvaluationCache.Clear();
290      testEvaluationCache.Clear();
291      evaluationCache.Clear();
292    }
293    public void RemoveClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
294      classificationSolutions.RemoveRange(solutions);
295
296      trainingEvaluationCache.Clear();
297      testEvaluationCache.Clear();
298      evaluationCache.Clear();
299    }
300
301    private void classificationSolutions_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
302      foreach (var solution in e.Items) AddClassificationSolution(solution);
303      RecalculateResults();
304    }
305    private void classificationSolutions_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
306      foreach (var solution in e.Items) RemoveClassificationSolution(solution);
307      RecalculateResults();
308    }
309    private void classificationSolutions_CollectionReset(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
310      foreach (var solution in e.OldItems) RemoveClassificationSolution(solution);
311      foreach (var solution in e.Items) AddClassificationSolution(solution);
312      RecalculateResults();
313    }
314
315    private void AddClassificationSolution(IClassificationSolution solution) {
316      if (Model.Models.Contains(solution.Model)) throw new ArgumentException();
317      Model.Add(solution.Model);
318      trainingPartitions[solution.Model] = solution.ProblemData.TrainingPartition;
319      testPartitions[solution.Model] = solution.ProblemData.TestPartition;
320
321      trainingEvaluationCache.Clear();
322      testEvaluationCache.Clear();
323      evaluationCache.Clear();
324    }
325
326    private void RemoveClassificationSolution(IClassificationSolution solution) {
327      if (!Model.Models.Contains(solution.Model)) throw new ArgumentException();
328      Model.Remove(solution.Model);
329      trainingPartitions.Remove(solution.Model);
330      testPartitions.Remove(solution.Model);
331
332      trainingEvaluationCache.Clear();
333      testEvaluationCache.Clear();
334      evaluationCache.Clear();
335    }
336  }
337}
Note: See TracBrowser for help on using the repository browser.