source: trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs @ 12816

Last change on this file since 12816 was 12816, checked in by gkronber, 7 years ago

#2448: added storable attribute to collection of individual solutions and recreate solutions in after-deserialization hook only when the collection is empty.

File size: 16.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Collections;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.Data;
29using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
30
31namespace HeuristicLab.Problems.DataAnalysis {
32  /// <summary>
33  /// Represents classification solutions that contain an ensemble of multiple classification models
34  /// </summary>
35  [StorableClass]
36  [Item("Classification Ensemble Solution", "A classification solution that contains an ensemble of multiple classification models")]
37  [Creatable(CreatableAttribute.Categories.DataAnalysisEnsembles, Priority = 110)]
38  public sealed class ClassificationEnsembleSolution : ClassificationSolutionBase, IClassificationEnsembleSolution {
39    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
40    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
41    private readonly Dictionary<int, double> evaluationCache = new Dictionary<int, double>();
42
43    public new IClassificationEnsembleModel Model {
44      get { return (IClassificationEnsembleModel)base.Model; }
45    }
46    public new ClassificationEnsembleProblemData ProblemData {
47      get { return (ClassificationEnsembleProblemData)base.ProblemData; }
48      set { base.ProblemData = value; }
49    }
50
51    [Storable]
52    private readonly ItemCollection<IClassificationSolution> classificationSolutions;
53    public IItemCollection<IClassificationSolution> ClassificationSolutions {
54      get { return classificationSolutions; }
55    }
56
57    [Storable]
58    private Dictionary<IClassificationModel, IntRange> trainingPartitions;
59    [Storable]
60    private Dictionary<IClassificationModel, IntRange> testPartitions;
61
62    [StorableConstructor]
63    private ClassificationEnsembleSolution(bool deserializing)
64      : base(deserializing) {
65      classificationSolutions = new ItemCollection<IClassificationSolution>();
66    }
67    [StorableHook(HookType.AfterDeserialization)]
68    private void AfterDeserialization() {
69      if (!classificationSolutions.Any()) {
70        foreach (var model in Model.Models) {
71          IClassificationProblemData problemData = (IClassificationProblemData)ProblemData.Clone();
72          problemData.TrainingPartition.Start = trainingPartitions[model].Start;
73          problemData.TrainingPartition.End = trainingPartitions[model].End;
74          problemData.TestPartition.Start = testPartitions[model].Start;
75          problemData.TestPartition.End = testPartitions[model].End;
76
77          classificationSolutions.Add(model.CreateClassificationSolution(problemData));
78        }
79      }
80      RegisterClassificationSolutionsEventHandler();
81    }
82
83    private ClassificationEnsembleSolution(ClassificationEnsembleSolution original, Cloner cloner)
84      : base(original, cloner) {
85      trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
86      testPartitions = new Dictionary<IClassificationModel, IntRange>();
87      foreach (var pair in original.trainingPartitions) {
88        trainingPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
89      }
90      foreach (var pair in original.testPartitions) {
91        testPartitions[cloner.Clone(pair.Key)] = cloner.Clone(pair.Value);
92      }
93
94      trainingEvaluationCache = new Dictionary<int, double>(original.ProblemData.TrainingIndices.Count());
95      testEvaluationCache = new Dictionary<int, double>(original.ProblemData.TestIndices.Count());
96
97      classificationSolutions = cloner.Clone(original.classificationSolutions);
98      RegisterClassificationSolutionsEventHandler();
99    }
100
101    public ClassificationEnsembleSolution()
102      : base(new ClassificationEnsembleModel(), ClassificationEnsembleProblemData.EmptyProblemData) {
103      trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
104      testPartitions = new Dictionary<IClassificationModel, IntRange>();
105      classificationSolutions = new ItemCollection<IClassificationSolution>();
106
107      RegisterClassificationSolutionsEventHandler();
108    }
109
110    public ClassificationEnsembleSolution(IClassificationProblemData problemData) :
111      this(Enumerable.Empty<IClassificationModel>(), problemData) { }
112
113    public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData)
114      : this(models, problemData,
115             models.Select(m => (IntRange)problemData.TrainingPartition.Clone()),
116             models.Select(m => (IntRange)problemData.TestPartition.Clone())
117      ) { }
118
119    public ClassificationEnsembleSolution(IEnumerable<IClassificationModel> models, IClassificationProblemData problemData, IEnumerable<IntRange> trainingPartitions, IEnumerable<IntRange> testPartitions)
120      : base(new ClassificationEnsembleModel(Enumerable.Empty<IClassificationModel>()), new ClassificationEnsembleProblemData(problemData)) {
121      this.trainingPartitions = new Dictionary<IClassificationModel, IntRange>();
122      this.testPartitions = new Dictionary<IClassificationModel, IntRange>();
123      this.classificationSolutions = new ItemCollection<IClassificationSolution>();
124
125      List<IClassificationSolution> solutions = new List<IClassificationSolution>();
126      var modelEnumerator = models.GetEnumerator();
127      var trainingPartitionEnumerator = trainingPartitions.GetEnumerator();
128      var testPartitionEnumerator = testPartitions.GetEnumerator();
129
130      while (modelEnumerator.MoveNext() & trainingPartitionEnumerator.MoveNext() & testPartitionEnumerator.MoveNext()) {
131        var p = (IClassificationProblemData)problemData.Clone();
132        p.TrainingPartition.Start = trainingPartitionEnumerator.Current.Start;
133        p.TrainingPartition.End = trainingPartitionEnumerator.Current.End;
134        p.TestPartition.Start = testPartitionEnumerator.Current.Start;
135        p.TestPartition.End = testPartitionEnumerator.Current.End;
136
137        solutions.Add(modelEnumerator.Current.CreateClassificationSolution(p));
138      }
139      if (modelEnumerator.MoveNext() | trainingPartitionEnumerator.MoveNext() | testPartitionEnumerator.MoveNext()) {
140        throw new ArgumentException();
141      }
142
143      trainingEvaluationCache = new Dictionary<int, double>(problemData.TrainingIndices.Count());
144      testEvaluationCache = new Dictionary<int, double>(problemData.TestIndices.Count());
145
146      RegisterClassificationSolutionsEventHandler();
147      classificationSolutions.AddRange(solutions);
148    }
149
150    public override IDeepCloneable Clone(Cloner cloner) {
151      return new ClassificationEnsembleSolution(this, cloner);
152    }
153    private void RegisterClassificationSolutionsEventHandler() {
154      classificationSolutions.ItemsAdded += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_ItemsAdded);
155      classificationSolutions.ItemsRemoved += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_ItemsRemoved);
156      classificationSolutions.CollectionReset += new CollectionItemsChangedEventHandler<IClassificationSolution>(classificationSolutions_CollectionReset);
157    }
158
159
160    #region Evaluation
161    public override IEnumerable<double> EstimatedClassValues {
162      get { return GetEstimatedClassValues(Enumerable.Range(0, ProblemData.Dataset.Rows)); }
163    }
164
165    public override IEnumerable<double> EstimatedTrainingClassValues {
166      get {
167        var rows = ProblemData.TrainingIndices;
168        var rowsToEvaluate = rows.Except(trainingEvaluationCache.Keys);
169        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
170        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, (r, m) => RowIsTrainingForModel(r, m) && !RowIsTestForModel(r, m)).GetEnumerator();
171
172        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
173          trainingEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
174        }
175
176        return rows.Select(row => trainingEvaluationCache[row]);
177      }
178    }
179
180    public override IEnumerable<double> EstimatedTestClassValues {
181      get {
182        var rows = ProblemData.TestIndices;
183        var rowsToEvaluate = rows.Except(testEvaluationCache.Keys);
184        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
185        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, RowIsTestForModel).GetEnumerator();
186
187        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
188          testEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
189        }
190
191        return rows.Select(row => testEvaluationCache[row]);
192      }
193    }
194
195    private IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows, Func<int, IClassificationModel, bool> modelSelectionPredicate) {
196      var estimatedValuesEnumerators = (from model in Model.Models
197                                        select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedClassValues(ProblemData.Dataset, rows).GetEnumerator() })
198                                       .ToList();
199      var rowsEnumerator = rows.GetEnumerator();
200      // aggregate to make sure that MoveNext is called for all enumerators
201      while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
202        int currentRow = rowsEnumerator.Current;
203
204        var selectedEnumerators = from pair in estimatedValuesEnumerators
205                                  where modelSelectionPredicate(currentRow, pair.Model)
206                                  select pair.EstimatedValuesEnumerator;
207
208        yield return AggregateEstimatedClassValues(selectedEnumerators.Select(x => x.Current));
209      }
210    }
211
212    private bool RowIsTrainingForModel(int currentRow, IClassificationModel model) {
213      return trainingPartitions == null || !trainingPartitions.ContainsKey(model) ||
214              (trainingPartitions[model].Start <= currentRow && currentRow < trainingPartitions[model].End);
215    }
216
217    private bool RowIsTestForModel(int currentRow, IClassificationModel model) {
218      return testPartitions == null || !testPartitions.ContainsKey(model) ||
219              (testPartitions[model].Start <= currentRow && currentRow < testPartitions[model].End);
220    }
221
222    public override IEnumerable<double> GetEstimatedClassValues(IEnumerable<int> rows) {
223      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
224      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
225      var valuesEnumerator = (from xs in GetEstimatedClassValueVectors(ProblemData.Dataset, rowsToEvaluate)
226                              select AggregateEstimatedClassValues(xs))
227                             .GetEnumerator();
228
229      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
230        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
231      }
232
233      return rows.Select(row => evaluationCache[row]);
234    }
235
236    public IEnumerable<IEnumerable<double>> GetEstimatedClassValueVectors(IDataset dataset, IEnumerable<int> rows) {
237      if (!Model.Models.Any()) yield break;
238      var estimatedValuesEnumerators = (from model in Model.Models
239                                        select model.GetEstimatedClassValues(dataset, rows).GetEnumerator())
240                                       .ToList();
241
242      while (estimatedValuesEnumerators.All(en => en.MoveNext())) {
243        yield return from enumerator in estimatedValuesEnumerators
244                     select enumerator.Current;
245      }
246    }
247
248    private double AggregateEstimatedClassValues(IEnumerable<double> estimatedClassValues) {
249      return estimatedClassValues
250      .GroupBy(x => x)
251      .OrderBy(g => -g.Count())
252      .Select(g => g.Key)
253      .DefaultIfEmpty(double.NaN)
254      .First();
255    }
256    #endregion
257
258    protected override void OnProblemDataChanged() {
259      trainingEvaluationCache.Clear();
260      testEvaluationCache.Clear();
261      evaluationCache.Clear();
262
263      IClassificationProblemData problemData = new ClassificationProblemData(ProblemData.Dataset,
264                                                                     ProblemData.AllowedInputVariables,
265                                                                     ProblemData.TargetVariable);
266      problemData.TrainingPartition.Start = ProblemData.TrainingPartition.Start;
267      problemData.TrainingPartition.End = ProblemData.TrainingPartition.End;
268      problemData.TestPartition.Start = ProblemData.TestPartition.Start;
269      problemData.TestPartition.End = ProblemData.TestPartition.End;
270
271      foreach (var solution in ClassificationSolutions) {
272        if (solution is ClassificationEnsembleSolution)
273          solution.ProblemData = ProblemData;
274        else
275          solution.ProblemData = problemData;
276      }
277      foreach (var trainingPartition in trainingPartitions.Values) {
278        trainingPartition.Start = ProblemData.TrainingPartition.Start;
279        trainingPartition.End = ProblemData.TrainingPartition.End;
280      }
281      foreach (var testPartition in testPartitions.Values) {
282        testPartition.Start = ProblemData.TestPartition.Start;
283        testPartition.End = ProblemData.TestPartition.End;
284      }
285
286      base.OnProblemDataChanged();
287    }
288
289    public void AddClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
290      classificationSolutions.AddRange(solutions);
291
292      trainingEvaluationCache.Clear();
293      testEvaluationCache.Clear();
294      evaluationCache.Clear();
295    }
296    public void RemoveClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
297      classificationSolutions.RemoveRange(solutions);
298
299      trainingEvaluationCache.Clear();
300      testEvaluationCache.Clear();
301      evaluationCache.Clear();
302    }
303
304    private void classificationSolutions_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
305      foreach (var solution in e.Items) AddClassificationSolution(solution);
306      RecalculateResults();
307    }
308    private void classificationSolutions_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
309      foreach (var solution in e.Items) RemoveClassificationSolution(solution);
310      RecalculateResults();
311    }
312    private void classificationSolutions_CollectionReset(object sender, CollectionItemsChangedEventArgs<IClassificationSolution> e) {
313      foreach (var solution in e.OldItems) RemoveClassificationSolution(solution);
314      foreach (var solution in e.Items) AddClassificationSolution(solution);
315      RecalculateResults();
316    }
317
318    private void AddClassificationSolution(IClassificationSolution solution) {
319      if (Model.Models.Contains(solution.Model)) throw new ArgumentException();
320      Model.Add(solution.Model);
321      trainingPartitions[solution.Model] = solution.ProblemData.TrainingPartition;
322      testPartitions[solution.Model] = solution.ProblemData.TestPartition;
323
324      trainingEvaluationCache.Clear();
325      testEvaluationCache.Clear();
326      evaluationCache.Clear();
327    }
328
329    private void RemoveClassificationSolution(IClassificationSolution solution) {
330      if (!Model.Models.Contains(solution.Model)) throw new ArgumentException();
331      Model.Remove(solution.Model);
332      trainingPartitions.Remove(solution.Model);
333      testPartitions.Remove(solution.Model);
334
335      trainingEvaluationCache.Clear();
336      testEvaluationCache.Clear();
337      evaluationCache.Clear();
338    }
339  }
340}
Note: See TracBrowser for help on using the repository browser.