source: trunk/sources/HeuristicLab.Clients.Hive/3.3/HiveTasks/OptimizerHiveTask.cs @ 8848

Last change on this file since 8848 was 8848, checked in by ascheibe, 10 years ago

#1950 fixed an InvalidOperationException in the Hive Job Manager

File size: 18.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Clients.Hive.Jobs;
26using HeuristicLab.Collections;
27using HeuristicLab.Common;
28using HeuristicLab.Optimization;
29using HeuristicLab.PluginInfrastructure;
30
31namespace HeuristicLab.Clients.Hive {
32  public class OptimizerHiveTask : HiveTask<OptimizerTask> {
33
34    Object batchRunLocker = new Object();
35
36    #region Constructors and Cloning
37    public OptimizerHiveTask() { }
38    public OptimizerHiveTask(IOptimizer optimizer)
39      : this() {
40      this.ItemTask = new OptimizerTask(optimizer);
41    }
42    public OptimizerHiveTask(OptimizerTask optimizerJob)
43      : this() {
44      this.ItemTask = optimizerJob;
45    }
46    protected OptimizerHiveTask(OptimizerHiveTask original, Cloner cloner)
47      : base(original, cloner) {
48    }
49    public override IDeepCloneable Clone(Cloner cloner) {
50      return new OptimizerHiveTask(this, cloner);
51    }
52    #endregion
53
54    /// <summary>
55    /// if this.Optimizer is an experiment
56    ///   Uses the child-optimizers of this.HiveTask and creates HiveTask-childs
57    /// if this.Optimizer is a batchrun
58    ///   Creates a number of child-jobs according to repetitions
59    /// </summary>
60    protected override void UpdateChildHiveTasks() {
61      base.UpdateChildHiveTasks();
62      if (Task != null && syncTasksWithOptimizers) {
63        if (!ItemTask.ComputeInParallel) {
64          this.childHiveTasks.Clear();
65        } else {
66          if (ItemTask.Item is Optimization.Experiment) {
67            Optimization.Experiment experiment = (Optimization.Experiment)ItemTask.Item;
68            foreach (IOptimizer childOpt in experiment.Optimizers) {
69              var optimizerHiveTask = new OptimizerHiveTask(childOpt);
70              optimizerHiveTask.Task.Priority = Task.Priority; //inherit priority from parent
71              this.childHiveTasks.Add(optimizerHiveTask);
72            }
73          } else if (ItemTask.Item is Optimization.BatchRun) {
74            Optimization.BatchRun batchRun = ItemTask.OptimizerAsBatchRun;
75            if (batchRun.Optimizer != null) {
76              while (this.childHiveTasks.Count < batchRun.Repetitions) {
77                var optimizerHiveTask = new OptimizerHiveTask(batchRun.Optimizer);
78                optimizerHiveTask.Task.Priority = Task.Priority;
79                this.childHiveTasks.Add(optimizerHiveTask);
80              }
81              while (this.childHiveTasks.Count > batchRun.Repetitions) {
82                this.childHiveTasks.Remove(this.childHiveTasks.Last());
83              }
84            }
85          }
86        }
87      }
88    }
89
90    protected override void RegisterItemTaskEvents() {
91      base.RegisterItemTaskEvents();
92      if (ItemTask != null) {
93        if (ItemTask.Item is Optimization.Experiment) {
94          Optimization.Experiment experiment = ItemTask.OptimizerAsExperiment;
95          experiment.Optimizers.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
96          experiment.Optimizers.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
97          experiment.Optimizers.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
98          experiment.Optimizers.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
99        } else if (ItemTask.Item is Optimization.BatchRun) {
100          Optimization.BatchRun batchRun = ItemTask.OptimizerAsBatchRun;
101          batchRun.RepetitionsChanged += new EventHandler(batchRun_RepetitionsChanged);
102          batchRun.OptimizerChanged += new EventHandler(batchRun_OptimizerChanged);
103        }
104      }
105    }
106    protected override void DergisterItemTaskEvents() {
107      base.DergisterItemTaskEvents();
108      if (ItemTask != null) {
109        if (ItemTask.Item is Optimization.Experiment) {
110          Optimization.Experiment experiment = ItemTask.OptimizerAsExperiment;
111          experiment.Optimizers.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
112          experiment.Optimizers.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
113          experiment.Optimizers.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
114          experiment.Optimizers.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
115        } else if (ItemTask.Item is Optimization.BatchRun) {
116          Optimization.BatchRun batchRun = ItemTask.OptimizerAsBatchRun;
117          batchRun.RepetitionsChanged -= new EventHandler(batchRun_RepetitionsChanged);
118          batchRun.OptimizerChanged -= new EventHandler(batchRun_OptimizerChanged);
119        }
120      }
121    }
122
123    private void batchRun_OptimizerChanged(object sender, EventArgs e) {
124      if (syncTasksWithOptimizers) {
125        UpdateChildHiveTasks();
126      }
127    }
128
129    private void batchRun_RepetitionsChanged(object sender, EventArgs e) {
130      if (syncTasksWithOptimizers) {
131        UpdateChildHiveTasks();
132      }
133    }
134
135    private void Optimizers_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
136      if (syncTasksWithOptimizers && this.ItemTask.ComputeInParallel) {
137        childHiveTasksLock.EnterWriteLock();
138        try {
139          foreach (var item in e.Items) {
140            if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
141              this.childHiveTasks.Add(new OptimizerHiveTask(item.Value));
142            }
143          }
144        }
145        finally { childHiveTasksLock.ExitWriteLock(); }
146      }
147    }
148    private void Optimizers_ItemsReplaced(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
149      if (syncTasksWithOptimizers && this.ItemTask.ComputeInParallel) {
150        childHiveTasksLock.EnterWriteLock();
151        try {
152          foreach (var item in e.OldItems) {
153            this.childHiveTasks.Remove(this.GetChildByOptimizer(item.Value));
154          }
155          foreach (var item in e.Items) {
156            if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
157              this.childHiveTasks.Add(new OptimizerHiveTask(item.Value));
158            }
159          }
160        }
161        finally { childHiveTasksLock.ExitWriteLock(); }
162      }
163    }
164    private void Optimizers_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
165      if (syncTasksWithOptimizers && this.ItemTask.ComputeInParallel) {
166        childHiveTasksLock.EnterWriteLock();
167        try {
168          foreach (var item in e.Items) {
169            this.childHiveTasks.Remove(this.GetChildByOptimizer(item.Value));
170          }
171        }
172        finally { childHiveTasksLock.ExitWriteLock(); }
173      }
174    }
175    private void Optimizers_CollectionReset(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
176      if (syncTasksWithOptimizers && this.ItemTask.ComputeInParallel) {
177        childHiveTasksLock.EnterWriteLock();
178        try {
179          foreach (var item in e.Items) {
180            this.childHiveTasks.Remove(this.GetChildByOptimizer(item.Value));
181          }
182        }
183        finally { childHiveTasksLock.ExitWriteLock(); }
184      }
185    }
186
187    /// <summary>
188    /// if this.Optimizer is Experiment
189    ///   replace the child-optimizer in the experiment
190    /// if this.Optimizer is BatchRun
191    ///   add the runs from the optimizerTask to the batchrun and replace the Optimizer
192    /// </summary>
193    public override void IntegrateChild(ItemTask task, Guid childTaskId) {
194      var optimizerTask = (OptimizerTask)task;
195      syncTasksWithOptimizers = false; // don't sync with optimizers during this method
196
197      if (this.ItemTask != null && this.ItemTask.Item != null) {
198        if (this.ItemTask.Item is Optimization.Experiment) {
199          UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerTask);
200        } else if (this.ItemTask.Item is Optimization.BatchRun) {
201          UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerTask);
202        }
203      }
204
205      IEnumerable<HiveTask> childs = this.ChildHiveTasks.Where(j => j.Task.Id == childTaskId);
206      //TODO: in very rare cases childs is empty. This shouldn't be the case and should be further investigated.
207      if (childs.Count() > 0) {
208        OptimizerHiveTask child = childs.First() as OptimizerHiveTask;
209
210        if (child != null && !optimizerTask.ComputeInParallel) {
211          child.syncTasksWithOptimizers = false;
212          child.ItemTask = optimizerTask;
213          child.syncTasksWithOptimizers = true;
214        }
215      }
216
217      syncTasksWithOptimizers = true;
218    }
219
220    /// <summary>
221    /// Adds the runs from the optimizerTask to the batchrun and replaces the Optimizer
222    /// Sideeffect: the optimizerTask.Optimizer will be prepared (scopes are deleted and executionstate will be reset)
223    /// </summary>
224    private void UpdateOptimizerInBatchRun(BatchRun batchRun, OptimizerTask optimizerTask) {
225      if (batchRun.Optimizer == null) {
226        batchRun.Optimizer = (IOptimizer)optimizerTask.Item; // only set the first optimizer as Optimizer. if every time the Optimizer would be set, the runs would be cleared each time
227      }
228      lock (batchRunLocker) {
229        foreach (IRun run in optimizerTask.Item.Runs) {
230          if (!batchRun.Runs.Contains(run)) {
231            run.Name = GetNewRunName(run, batchRun.Runs);
232            batchRun.Runs.Add(run);
233          }
234        }
235      }
236    }
237
238    /// <summary>
239    /// replace the child-optimizer in the experiment
240    /// Sideeffect: the optimizerTask.Optimizer will be prepared (scopes are deleted and executionstate will be reset)
241    /// </summary>
242    private void UpdateOptimizerInExperiment(Optimization.Experiment experiment, OptimizerTask optimizerTask) {
243      if (optimizerTask.IndexInParentOptimizerList < 0)
244        throw new IndexOutOfRangeException("IndexInParentOptimizerList must be equal or greater than zero! The Task is invalid and the optimizer-tree cannot be reassembled.");
245
246      while (experiment.Optimizers.Count < optimizerTask.IndexInParentOptimizerList) {
247        experiment.Optimizers.Add(new UserDefinedAlgorithm("Placeholder")); // add dummy-entries to Optimizers so that its possible to insert the optimizerTask at the correct position
248      }
249      if (experiment.Optimizers.Count < optimizerTask.IndexInParentOptimizerList + 1) {
250        experiment.Optimizers.Add(optimizerTask.Item);
251      } else {
252        // if ComputeInParallel==true, don't replace the optimizer (except it is still a Placeholder)
253        // this is because Jobs with ComputeInParallel get submitted to hive with their child-optimizers deleted
254        if (!optimizerTask.ComputeInParallel || experiment.Optimizers[optimizerTask.IndexInParentOptimizerList].Name == "Placeholder") {
255          experiment.Optimizers[optimizerTask.IndexInParentOptimizerList] = optimizerTask.Item;
256        }
257      }
258    }
259
260    /// <summary>
261    /// Sets the IndexInParentOptimizerList property of the OptimizerJob
262    /// according to the position in the OptimizerList of the parentHiveTask.Task
263    /// Recursively updates all the child-jobs as well
264    /// </summary>
265    internal void SetIndexInParentOptimizerList(OptimizerHiveTask parentHiveTask) {
266      if (parentHiveTask != null) {
267        if (parentHiveTask.ItemTask.Item is Optimization.Experiment) {
268          this.ItemTask.IndexInParentOptimizerList = parentHiveTask.ItemTask.OptimizerAsExperiment.Optimizers.IndexOf(this.ItemTask.Item);
269        } else if (parentHiveTask.ItemTask.Item is Optimization.BatchRun) {
270          this.ItemTask.IndexInParentOptimizerList = 0;
271        } else {
272          throw new NotSupportedException("Only Experiment and BatchRuns are supported");
273        }
274      }
275      childHiveTasksLock.EnterReadLock();
276      try {
277        foreach (OptimizerHiveTask child in childHiveTasks) {
278          child.SetIndexInParentOptimizerList(this);
279        }
280      }
281      finally { childHiveTasksLock.ExitReadLock(); }
282    }
283
284    public override void AddChildHiveTask(HiveTask hiveTask) {
285      base.AddChildHiveTask(hiveTask);
286      var optimizerHiveJob = (OptimizerHiveTask)hiveTask;
287      syncTasksWithOptimizers = false;
288      if (this.ItemTask != null && optimizerHiveJob.ItemTask != null) {
289        // if task is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment
290        OptimizerTask optimizerJobClone = null;
291        if (optimizerHiveJob.Task.State == TaskState.Paused) {
292          optimizerJobClone = (OptimizerTask)optimizerHiveJob.ItemTask.Clone();
293        }
294
295        if (this.ItemTask.Item is Optimization.Experiment) {
296          if (!this.ItemTask.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.ItemTask.Item)) {
297            UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerHiveJob.ItemTask);
298          }
299        } else if (this.ItemTask.Item is Optimization.BatchRun) {
300          UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerHiveJob.ItemTask);
301        }
302
303        if (optimizerHiveJob.Task.State == TaskState.Paused) {
304          optimizerHiveJob.ItemTask = optimizerJobClone;
305        }
306      }
307      syncTasksWithOptimizers = true;
308    }
309
310    /// <summary>
311    /// Creates a TaskData object containing the Task and the IJob-Object as byte[]
312    /// </summary>
313    /// <param name="withoutChildOptimizers">
314    ///   if true the Child-Optimizers will not be serialized (if the task contains an Experiment)
315    /// </param>
316    public override TaskData GetAsTaskData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
317      plugins = new List<IPluginDescription>();
318      if (this.itemTask == null) // || this.jobItem.Optimizer == null
319        return null;
320
321      IEnumerable<Type> usedTypes;
322      byte[] jobByteArray;
323      if (withoutChildOptimizers && this.ItemTask.Item is Optimization.Experiment) {
324        OptimizerTask clonedJob = (OptimizerTask)this.ItemTask.Clone(); // use a cloned task, so that the childHiveJob don't get confused
325        clonedJob.OptimizerAsExperiment.Optimizers.Clear();
326        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
327      } else if (withoutChildOptimizers && this.ItemTask.Item is Optimization.BatchRun) {
328        OptimizerTask clonedJob = (OptimizerTask)this.ItemTask.Clone();
329        clonedJob.OptimizerAsBatchRun.Optimizer = null;
330        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
331      } else if (this.ItemTask.Item is IAlgorithm) {
332        ((IAlgorithm)this.ItemTask.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
333        jobByteArray = PersistenceUtil.Serialize(this.ItemTask, out usedTypes);
334      } else {
335        jobByteArray = PersistenceUtil.Serialize(this.ItemTask, out usedTypes);
336      }
337
338      TaskData jobData = new TaskData() { TaskId = task.Id, Data = jobByteArray };
339      PluginUtil.CollectDeclaringPlugins(plugins, usedTypes);
340      return jobData;
341    }
342
343    public OptimizerHiveTask GetChildByOptimizerTask(OptimizerTask optimizerTask) {
344      childHiveTasksLock.EnterReadLock();
345      try {
346        foreach (OptimizerHiveTask child in childHiveTasks) {
347          if (child.ItemTask == optimizerTask)
348            return child;
349        }
350        return null;
351      }
352      finally { childHiveTasksLock.ExitReadLock(); }
353    }
354
355    public HiveTask<OptimizerTask> GetChildByOptimizer(IOptimizer optimizer) {
356      childHiveTasksLock.EnterReadLock();
357      try {
358        foreach (OptimizerHiveTask child in childHiveTasks) {
359          if (child.ItemTask.Item == optimizer)
360            return child;
361        }
362        return null;
363      }
364      finally { childHiveTasksLock.ExitReadLock(); }
365    }
366
367    public RunCollection GetRuns() {
368      childHiveTasksLock.EnterReadLock();
369      try {
370        return this.ItemTask.Item.Runs;
371      }
372      finally {
373        childHiveTasksLock.ExitReadLock();
374      }
375    }
376
377    #region Helpers
378    /// <summary>
379    /// Parses the run numbers out of runs and renames the run to the next number
380    /// </summary>
381    private static string GetNewRunName(IRun run, RunCollection runs) {
382      int idx = run.Name.IndexOf("Run ") + 4;
383
384      if (idx == 3 || runs.Count == 0)
385        return run.Name;
386
387      int maxRunNumber = int.MinValue;
388      foreach (IRun r in runs) {
389        int number = GetRunNumber(r.Name);
390        maxRunNumber = Math.Max(maxRunNumber, number);
391      }
392
393      return run.Name.Substring(0, idx) + (maxRunNumber + 1).ToString();
394    }
395
396    /// <summary>
397    /// Parses the number of a Run out of its name. Example "Genetic Algorithm Run 3" -> 3
398    /// </summary>
399    private static int GetRunNumber(string runName) {
400      int idx = runName.IndexOf("Run ") + 4;
401      if (idx == 3) {
402        return 0;
403      } else {
404        return int.Parse(runName.Substring(idx, runName.Length - idx));
405      }
406    }
407    #endregion
408  }
409}
Note: See TracBrowser for help on using the repository browser.