source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/OptimizerHiveJob.cs @ 6033

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Clients.Hive.Jobs;
using HeuristicLab.Collections;
using HeuristicLab.Common;
using HeuristicLab.Optimization;
using HeuristicLab.PluginInfrastructure;

namespace HeuristicLab.Clients.Hive.ExperimentManager {
  public class OptimizerHiveJob : HiveJob<OptimizerJob> {

    #region Constructors and Cloning
    public OptimizerHiveJob() { }
    public OptimizerHiveJob(IOptimizer optimizer)
      : this() {
      this.JobItem = new OptimizerJob(optimizer);
    }
    public OptimizerHiveJob(OptimizerJob optimizerJob)
      : this() {
      this.JobItem = optimizerJob;
    }
    protected OptimizerHiveJob(OptimizerHiveJob original, Cloner cloner)
      : base(original, cloner) {
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new OptimizerHiveJob(this, cloner);
    }
    #endregion

    /// <summary>
    /// if this.Optimizer is an experiment
    ///   Uses the child-optimizers of this.HiveJob and creates HiveJob-childs
    /// if this.Optimizer is a batchrun
    ///   Creates a number of child-jobs according to repetitions
    /// </summary>
    protected override void UpdateChildHiveJobs() {
      base.UpdateChildHiveJobs();
      if (Job != null && syncJobsWithOptimizers) {
        if (!JobItem.ComputeInParallel) {
          this.childHiveJobs.Clear();
        } else {
          if (JobItem.Item is Optimization.Experiment) {
            Optimization.Experiment experiment = (Optimization.Experiment)JobItem.Item;
            foreach (IOptimizer childOpt in experiment.Optimizers) {
              this.childHiveJobs.Add(new OptimizerHiveJob(childOpt));
            }
          } else if (JobItem.Item is Optimization.BatchRun) {
            Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
            if (batchRun.Optimizer != null) {
              while (this.childHiveJobs.Count < batchRun.Repetitions) {
                this.childHiveJobs.Add(new OptimizerHiveJob(batchRun.Optimizer));
              }
              while (this.childHiveJobs.Count > batchRun.Repetitions) {
                this.childHiveJobs.Remove(this.childHiveJobs.Last());
              }
            }
          }
        }
      }
    }

    protected override void RegisterJobItemEvents() {
      base.RegisterJobItemEvents();
      if (JobItem != null) {
        if (JobItem.Item is Optimization.Experiment) {
          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
          experiment.Optimizers.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
          experiment.Optimizers.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
          experiment.Optimizers.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
          experiment.Optimizers.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
        } else if (JobItem.Item is Optimization.BatchRun) {
          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
          batchRun.RepetitionsChanged += new EventHandler(batchRun_RepetitionsChanged);
          batchRun.OptimizerChanged += new EventHandler(batchRun_OptimizerChanged);
        }
      }
    }
    protected override void DergisterJobItemsEvents() {
      base.DergisterJobItemsEvents();
      if (JobItem != null) {
        if (JobItem.Item is Optimization.Experiment) {
          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
          experiment.Optimizers.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
          experiment.Optimizers.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
          experiment.Optimizers.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
          experiment.Optimizers.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
        } else if (JobItem.Item is Optimization.BatchRun) {
          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
          batchRun.RepetitionsChanged -= new EventHandler(batchRun_RepetitionsChanged);
          batchRun.OptimizerChanged -= new EventHandler(batchRun_OptimizerChanged);
        }
      }
    }

    private void batchRun_OptimizerChanged(object sender, EventArgs e) {
      if (syncJobsWithOptimizers) {
        UpdateChildHiveJobs();
      }
    }

    private void batchRun_RepetitionsChanged(object sender, EventArgs e) {
      if (syncJobsWithOptimizers) {
        UpdateChildHiveJobs();
      }
    }

    private void Optimizers_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
        foreach (var item in e.Items) {
          if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
            this.childHiveJobs.Add(new OptimizerHiveJob(item.Value));
          }
        }
      }
    }
    private void Optimizers_ItemsReplaced(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
        foreach (var item in e.OldItems) {
          this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
        }
        foreach (var item in e.Items) {
          if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
            this.childHiveJobs.Add(new OptimizerHiveJob(item.Value));
          }
        }
      }
    }
    private void Optimizers_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
        foreach (var item in e.Items) {
          this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
        }
      }
    }
    private void Optimizers_CollectionReset(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
        foreach (var item in e.Items) {
          this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
        }
      }
    }

    /// <summary>
    /// if this.Optimizer is Experiment
    ///   replace the child-optimizer in the experiment
    /// if this.Optimizer is BatchRun
    ///   add the runs from the optimizerJob to the batchrun and replace the Optimizer
    /// </summary>
    public void UpdateChildOptimizer(OptimizerJob optimizerJob, Guid childJobId) {
      syncJobsWithOptimizers = false; // don't sync with optimizers during this method

      if (this.JobItem != null && this.JobItem.Item != null) {
        if (this.JobItem.Item is Optimization.Experiment) {
          UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerJob);
        } else if (this.JobItem.Item is Optimization.BatchRun) {
          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerJob);
        }
      }

      OptimizerHiveJob child = (OptimizerHiveJob)this.ChildHiveJobs.Single(j => j.Job.Id == childJobId);
      if (!optimizerJob.ComputeInParallel) {
        child.syncJobsWithOptimizers = false;
        child.JobItem = optimizerJob;
        child.syncJobsWithOptimizers = true;
      }
      syncJobsWithOptimizers = true;
    }

    /// <summary>
    /// Adds the runs from the optimizerJob to the batchrun and replaces the Optimizer 
    /// Sideeffect: the optimizerJob.Optimizer will be prepared (scopes are deleted and executionstate will be reset)
    /// </summary>
    private void UpdateOptimizerInBatchRun(BatchRun batchRun, OptimizerJob optimizerJob) {
      if (batchRun.Optimizer == null) {
        batchRun.Optimizer = (IOptimizer)optimizerJob.Item; // only set the first optimizer as Optimizer. if every time the Optimizer would be set, the runs would be cleared each time
      }
      foreach (IRun run in optimizerJob.Item.Runs) {
        if (!batchRun.Runs.Contains(run)) {
          run.Name = GetNewRunName(run, batchRun.Runs);
          batchRun.Runs.Add(run);
        }
      }
    }

    /// <summary>
    /// replace the child-optimizer in the experiment
    /// Sideeffect: the optimizerJob.Optimizer will be prepared (scopes are deleted and executionstate will be reset)
    /// </summary>
    private void UpdateOptimizerInExperiment(Optimization.Experiment experiment, OptimizerJob optimizerJob) {
      if (optimizerJob.IndexInParentOptimizerList < 0)
        throw new IndexOutOfRangeException("IndexInParentOptimizerList must be equal or greater than zero! The Job is invalid and the optimizer-tree cannot be reassembled.");

      while (experiment.Optimizers.Count < optimizerJob.IndexInParentOptimizerList) {
        experiment.Optimizers.Add(new UserDefinedAlgorithm("Placeholder")); // add dummy-entries to Optimizers so that its possible to insert the optimizerJob at the correct position
      }
      if (experiment.Optimizers.Count < optimizerJob.IndexInParentOptimizerList + 1) {
        experiment.Optimizers.Add(optimizerJob.Item);
      } else {
        // if ComputeInParallel==true, don't replace the optimizer (except it is still a Placeholder)
        // this is because Jobs with ComputeInParallel get submitted to hive with their child-optimizers deleted
        if (!optimizerJob.ComputeInParallel || experiment.Optimizers[optimizerJob.IndexInParentOptimizerList].Name == "Placeholder") {
          experiment.Optimizers[optimizerJob.IndexInParentOptimizerList] = optimizerJob.Item;
        }
      }
    }

    /// <summary>
    /// Sets the IndexInParentOptimizerList property of the OptimizerJob
    /// according to the position in the OptimizerList of the parentHiveJob.Job
    /// Recursively updates all the child-jobs as well
    /// </summary>
    internal void SetIndexInParentOptimizerList(OptimizerHiveJob parentHiveJob) {
      if (parentHiveJob != null) {
        if (parentHiveJob.JobItem.Item is Optimization.Experiment) {
          this.JobItem.IndexInParentOptimizerList = parentHiveJob.JobItem.OptimizerAsExperiment.Optimizers.IndexOf(this.JobItem.Item);
        } else if (parentHiveJob.JobItem.Item is Optimization.BatchRun) {
          this.JobItem.IndexInParentOptimizerList = 0;
        } else {
          throw new NotSupportedException("Only Experiment and BatchRuns are supported");
        }
      }
      foreach (OptimizerHiveJob child in childHiveJobs) {
        child.SetIndexInParentOptimizerList(this);
      }
    }

    public override void AddChildHiveJob(HiveJob hiveJob) {
      base.AddChildHiveJob(hiveJob);
      var optimizerHiveJob = (OptimizerHiveJob)hiveJob;
      syncJobsWithOptimizers = false;
      if (this.JobItem != null && optimizerHiveJob.JobItem != null) {
        // if job is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment
        OptimizerJob optimizerJobClone = null;
        if (optimizerHiveJob.Job.State == JobState.Paused) {
          optimizerJobClone = (OptimizerJob)optimizerHiveJob.JobItem.Clone();
        }

        if (this.JobItem.Item is Optimization.Experiment) {
          if (!this.JobItem.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.JobItem.Item)) {
            UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerHiveJob.JobItem);
          }
        } else if (this.JobItem.Item is Optimization.BatchRun) {
          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerHiveJob.JobItem);
        }

        if (optimizerHiveJob.Job.State == JobState.Paused) {
          optimizerHiveJob.JobItem = optimizerJobClone;
        }
      }
      syncJobsWithOptimizers = true;
    }

    /// <summary>
    /// Creates a JobData object containing the Job and the IJob-Object as byte[]
    /// </summary>
    /// <param name="withoutChildOptimizers">
    ///   if true the Child-Optimizers will not be serialized (if the job contains an Experiment)
    /// </param>
    public override JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
      plugins = new List<IPluginDescription>();
      if (this.jobItem == null) // || this.jobItem.Optimizer == null
        return null;

      IEnumerable<Type> usedTypes;
      byte[] jobByteArray;
      if (withoutChildOptimizers && this.JobItem.Item is Optimization.Experiment) {
        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone(); // use a cloned job, so that the childHiveJob don't get confused
        clonedJob.OptimizerAsExperiment.Optimizers.Clear();
        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
      } else if (withoutChildOptimizers && this.JobItem.Item is Optimization.BatchRun) {
        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone();
        clonedJob.OptimizerAsBatchRun.Optimizer = null;
        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
      } else if (this.JobItem.Item is IAlgorithm) {
        ((IAlgorithm)this.JobItem.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
      } else {
        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
      }

      JobData jobData = new JobData() {
        JobId = job.Id,
        Data = jobByteArray
      };

      PluginUtil.CollectDeclaringPlugins(plugins, usedTypes);

      return jobData;
    }

    public OptimizerHiveJob GetChildByOptimizerJob(OptimizerJob optimizerJob) {
      foreach (OptimizerHiveJob child in ChildHiveJobs) {
        if (child.JobItem == optimizerJob)
          return child;
      }
      return null;
    }

    public HiveJob<OptimizerJob> GetChildByOptimizer(IOptimizer optimizer) {
      foreach (OptimizerHiveJob child in ChildHiveJobs) {
        if (child.JobItem.Item == optimizer)
          return child;
      }
      return null;
    }

    #region Helpers
    /// <summary>
    /// Parses the run numbers out of runs and renames the run to the next number
    /// </summary>
    private static string GetNewRunName(IRun run, RunCollection runs) {
      int idx = run.Name.IndexOf("Run ") + 4;

      if (idx == -1 || runs.Count == 0)
        return run.Name;

      int maxRunNumber = int.MinValue;
      foreach (IRun r in runs) {
        int number = GetRunNumber(r.Name);
        maxRunNumber = Math.Max(maxRunNumber, number);
      }

      return run.Name.Substring(0, idx) + (maxRunNumber + 1).ToString();
    }

    /// <summary>
    /// Parses the number of a Run out of its name. Example "Genetic Algorithm Run 3" -> 3
    /// </summary>
    private static int GetRunNumber(string runName) {
      int idx = runName.IndexOf("Run ") + 4;
      if (idx == -1) {
        return 0;
      } else {
        return int.Parse(runName.Substring(idx, runName.Length - idx));
      }
    }
    #endregion
  }
}