Free cookie consent management tool by TermsFeed Policy Generator

Changeset 4139


Ignore:
Timestamp:
08/03/10 15:23:57 (13 years ago)
Author:
cneumuel
Message:

HiveExperiment now flats the optimizer-tree (supports batch runs and sub-experiments)

Location:
branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HeuristicLab.Hive.Experiment-3.3.csproj

    r4133 r4139  
    9696  </ItemGroup>
    9797  <ItemGroup>
     98    <Compile Include="OptimizerList.cs" />
    9899    <Compile Include="JobItemList.cs" />
    99100    <Compile Include="JobItem.cs" />
  • branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HiveExperiment.cs

    r4135 r4139  
    5151  public class HiveExperiment : NamedItem, IExecutable {
    5252    private const string itemName = "Hive Experiment";
    53     private const string itemDescription = "An experiment which contains multiple batch runs of algorithms which are executed in the Hive.";
     53    private const string itemDescription = "A runner for a single experiment, which's algorithms are executed in the Hive.";
    5454    private const int resultPollingIntervalMs = 15000;
    5555    private const int maxSnapshotRetries = 20;
     
    7676
    7777    private IDictionary<Guid, Thread> resultPollingThreads;
    78    
    79     [Storable]
    80     private IDictionary<Guid, IOptimizer> pendingOptimizers = new Dictionary<Guid, IOptimizer>();
     78
     79    /// <summary>
     80    /// Mapping from JobId to an optimizer.
     81    /// Stores all pending optimizers. If an optimizer is finished it is removed from this collection
     82    /// </summary>
     83    [Storable]
     84    private IDictionary<Guid, IOptimizer> pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
     85
     86    /// <summary>
     87    /// Stores a mapping from the child-optimizer to the parent optimizer.
     88    /// Needed to replace a finished optimizer in the optimizer-tree.
     89    /// Only pending optmizers are stored.
     90    /// </summary>
     91    [Storable]
     92    private IDictionary<IOptimizer, IOptimizer> parentOptimizersByPendingOptimizer = new Dictionary<IOptimizer, IOptimizer>();
    8193
    8294    [Storable]
     
    156168      clone.executionState = this.executionState;
    157169      clone.executionTime = this.executionTime;
    158       clone.pendingOptimizers = new Dictionary<Guid, IOptimizer>();
    159       foreach (var pair in this.pendingOptimizers)
    160         clone.pendingOptimizers[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
     170      clone.pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
     171      foreach (var pair in this.pendingOptimizersByJobId)
     172        clone.pendingOptimizersByJobId[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
     173      foreach (var pair in this.parentOptimizersByPendingOptimizer)
     174        clone.parentOptimizersByPendingOptimizer[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
    161175      clone.log = (ILog)cloner.Clone(log);
    162176      clone.stopPending = this.stopPending;
     
    242256        IExecutionEngineFacade executionEngineFacade = GetExecutionEngineFacade();
    243257
    244         pendingOptimizers = new Dictionary<Guid, IOptimizer>();
     258        pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
     259        parentOptimizersByPendingOptimizer = GetOptimizers(true);
    245260        IEnumerable<string> groups = ResourceGroups;
    246 
    247         foreach (IOptimizer optimizer in GetOptimizers(false)) {
     261       
     262        foreach (IOptimizer optimizer in parentOptimizersByPendingOptimizer.Keys) {
    248263          SerializedJob serializedJob = CreateSerializedJob(optimizer);
    249264          ResponseObject<JobDto> response = executionEngineFacade.AddJobWithGroupStrings(serializedJob, groups);
    250           pendingOptimizers.Add(response.Obj.Id, optimizer);
     265          pendingOptimizersByJobId.Add(response.Obj.Id, optimizer);
    251266
    252267          JobItem jobItem = new JobItem() {
     
    263278          LogMessage("Sent job to Hive (jobId: " + response.Obj.Id + ")");
    264279        }
    265        
     280
    266281        // start results polling after sending sending the jobs to the server (to avoid race conflicts at the optimizers-collection)
    267282        StartResultPolling();
     
    279294
    280295    private void CreateResultPollingThreads() {
    281       foreach(JobItem jobItem in JobItems) {
     296      foreach (JobItem jobItem in JobItems) {
    282297        if (!resultPollingThreads.ContainsKey(jobItem.JobDto.Id)) {
    283298          resultPollingThreads.Add(jobItem.JobDto.Id, CreateResultPollingThread(jobItem.JobDto));
     
    314329    /// <param name="flatout">if false only top level optimizers are returned, if true the optimizer-tree is flatted</param>
    315330    /// <returns></returns>
    316     private IEnumerable<IOptimizer> GetOptimizers(bool flatout) {
     331    private IDictionary<IOptimizer, IOptimizer> GetOptimizers(bool flatout) {
    317332      if (!flatout) {
    318         return experiment.Optimizers;
     333        var optimizers = new Dictionary<IOptimizer, IOptimizer>();
     334        foreach (IOptimizer opt in experiment.Optimizers) {
     335          optimizers.Add(experiment, opt);
     336        }
     337        return optimizers;
    319338      } else {
    320         throw new NotImplementedException();
    321       }
    322     }
    323 
    324     private void ReplaceOptimizer(IOptimizer originalOptimizer, IOptimizer newOptimizer) {
     339        return FlatOptimizerTree(null, experiment);
     340      }
     341    }
     342
     343    /// <summary>
     344    /// Recursively iterates all IOptimizers in the optimizer-tree and returns them.
     345    ///
     346    /// [chn] this could be implemented more cleanly if Experiment and BatchRun would implement an interface like:
     347    /// interface IParallelizable {
     348    ///   IEnumerable&lt;IOptimizer&gt; GetOptimizers();
     349    /// }
     350    /// </summary>
     351    /// <returns>a dictionary mapping from the parent optimizer to the child optimizer</returns>
     352    private IDictionary<IOptimizer, IOptimizer> FlatOptimizerTree(IOptimizer parent, IOptimizer optimizer) {
     353      IDictionary<IOptimizer, IOptimizer> optimizers = new Dictionary<IOptimizer, IOptimizer>();
     354      if (optimizer is HeuristicLab.Optimization.Experiment) {
     355        HeuristicLab.Optimization.Experiment experiment = optimizer as HeuristicLab.Optimization.Experiment;
     356        foreach (IOptimizer opt in experiment.Optimizers) {
     357          AddRange(optimizers, FlatOptimizerTree(experiment, opt));
     358        }
     359      } else if (optimizer is BatchRun) {
     360        BatchRun batchRun = optimizer as BatchRun;
     361        for (int i = 0; i < batchRun.Repetitions; i++) {
     362          IOptimizer opt = (IOptimizer)batchRun.Algorithm.Clone();
     363          AddRange(optimizers, FlatOptimizerTree(batchRun, opt));
     364        }
     365      } else if (optimizer is EngineAlgorithm) {
     366        optimizers.Add(optimizer, parent);
     367      } else {
     368        Logger.Warn("Optimizer of type " + optimizers.GetType().ToString() + " unknown");
     369        optimizers.Add(optimizer, parent);
     370      }
     371      return optimizers;
     372    }
     373
     374    private void AddRange(IDictionary<IOptimizer, IOptimizer> optimizers, IDictionary<IOptimizer, IOptimizer> childs) {
     375      foreach (KeyValuePair<IOptimizer, IOptimizer> kvp in childs) {
     376        optimizers.Add(kvp);
     377      }
     378    }
     379
     380    private void ReplaceOptimizer(IOptimizer parentOptimizer, IOptimizer originalOptimizer, IOptimizer newOptimizer) {
    325381      lock (locker) {
    326         int originalOptimizerIndex = experiment.Optimizers.IndexOf(originalOptimizer);
    327         experiment.Optimizers[originalOptimizerIndex] = newOptimizer;
     382        if (parentOptimizer is HeuristicLab.Optimization.Experiment) {
     383          HeuristicLab.Optimization.Experiment exp = (HeuristicLab.Optimization.Experiment)parentOptimizer;
     384          int originalOptimizerIndex = exp.Optimizers.IndexOf(originalOptimizer);
     385          exp.Optimizers[originalOptimizerIndex] = newOptimizer;
     386        } else if (parentOptimizer is BatchRun) {
     387          BatchRun batchRun = (BatchRun)parentOptimizer;
     388          if (newOptimizer is IAlgorithm) {
     389            batchRun.Runs.Add(new Run((IAlgorithm)newOptimizer));
     390          } else {
     391            throw new NotSupportedException("Only IAlgorithm types supported");
     392          }
     393        } else {
     394          throw new NotSupportedException("Invalid parentOptimizer");
     395        }
    328396      }
    329397    }
     
    399467              GetJobItemById(job.Id).LogMessage("Job successfully aborted");
    400468              return;
    401             }           
     469            }
    402470
    403471            // loop while
     
    416484          GetJobItemById(job.Id).LogMessage("Job finished");
    417485          // job retrieved... replace the existing optimizers with the finished one
    418           IOptimizer originalOptimizer = pendingOptimizers[job.Id];
     486          IOptimizer originalOptimizer = pendingOptimizersByJobId[job.Id];
    419487          IOptimizer restoredOptimizer = ((OptimizerJob)restoredObject).Optimizer;
    420488
    421           ReplaceOptimizer(originalOptimizer, restoredOptimizer);
    422           pendingOptimizers.Remove(job.Id);
    423 
    424           if (pendingOptimizers.Count == 0) {
     489          ReplaceOptimizer(parentOptimizersByPendingOptimizer[originalOptimizer], originalOptimizer, restoredOptimizer);
     490          pendingOptimizersByJobId.Remove(job.Id);
     491          parentOptimizersByPendingOptimizer.Remove(originalOptimizer);
     492
     493          if (pendingOptimizersByJobId.Count == 0) {
    425494            // finished
    426495            this.ExecutionState = Core.ExecutionState.Stopped;
Note: See TracChangeset for help on using the changeset viewer.