Changeset 4139

Timestamp:

08/03/10 15:23:57 (14 years ago)

Author:

cneumuel

Message:

HiveExperiment now flats the optimizer-tree (supports batch runs and sub-experiments)

Location:

branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3

Files:

• HeuristicLab.Hive.Experiment-3.3.csproj (modified) (1 diff)
• HiveExperiment.cs (modified) (9 diffs)

branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HeuristicLab.Hive.Experiment-3.3.csproj

@@ -96 +96 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="OptimizerList.cs" />
     <Compile Include="JobItemList.cs" />
     <Compile Include="JobItem.cs" />

branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HiveExperiment.cs

@@ -51 +51 @@
   public class HiveExperiment : NamedItem, IExecutable {
     private const string itemName = "Hive Experiment";
-    private const string itemDescription = "An experiment which contains multiple batch runs of algorithms which are executed in the Hive.";
+    private const string itemDescription = "A runner for a single experiment, which's algorithms are executed in the Hive.";
     private const int resultPollingIntervalMs = 15000;
     private const int maxSnapshotRetries = 20;
@@ -76 +76 @@
 
     private IDictionary<Guid, Thread> resultPollingThreads;
-    
-    [Storable]
-    private IDictionary<Guid, IOptimizer> pendingOptimizers = new Dictionary<Guid, IOptimizer>();
+
+    /// <summary>
+    /// Mapping from JobId to an optimizer.
+    /// Stores all pending optimizers. If an optimizer is finished it is removed from this collection
+    /// </summary>
+    [Storable]
+    private IDictionary<Guid, IOptimizer> pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
+
+    /// <summary>
+    /// Stores a mapping from the child-optimizer to the parent optimizer.
+    /// Needed to replace a finished optimizer in the optimizer-tree.
+    /// Only pending optmizers are stored.
+    /// </summary>
+    [Storable]
+    private IDictionary<IOptimizer, IOptimizer> parentOptimizersByPendingOptimizer = new Dictionary<IOptimizer, IOptimizer>();
 
     [Storable]
@@ -156 +168 @@
       clone.executionState = this.executionState;
       clone.executionTime = this.executionTime;
-      clone.pendingOptimizers = new Dictionary<Guid, IOptimizer>();
-      foreach (var pair in this.pendingOptimizers)
-        clone.pendingOptimizers[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
+      clone.pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
+      foreach (var pair in this.pendingOptimizersByJobId)
+        clone.pendingOptimizersByJobId[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
+      foreach (var pair in this.parentOptimizersByPendingOptimizer)
+        clone.parentOptimizersByPendingOptimizer[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
       clone.log = (ILog)cloner.Clone(log);
       clone.stopPending = this.stopPending;
@@ -242 +256 @@
         IExecutionEngineFacade executionEngineFacade = GetExecutionEngineFacade();
 
-        pendingOptimizers = new Dictionary<Guid, IOptimizer>();
+        pendingOptimizersByJobId = new Dictionary<Guid, IOptimizer>();
+        parentOptimizersByPendingOptimizer = GetOptimizers(true);
         IEnumerable<string> groups = ResourceGroups;
-
-        foreach (IOptimizer optimizer in GetOptimizers(false)) {
+        
+        foreach (IOptimizer optimizer in parentOptimizersByPendingOptimizer.Keys) {
           SerializedJob serializedJob = CreateSerializedJob(optimizer);
           ResponseObject<JobDto> response = executionEngineFacade.AddJobWithGroupStrings(serializedJob, groups);
-          pendingOptimizers.Add(response.Obj.Id, optimizer);
+          pendingOptimizersByJobId.Add(response.Obj.Id, optimizer);
 
           JobItem jobItem = new JobItem() {
@@ -263 +278 @@
           LogMessage("Sent job to Hive (jobId: " + response.Obj.Id + ")");
         }
-        
+
         // start results polling after sending sending the jobs to the server (to avoid race conflicts at the optimizers-collection)
         StartResultPolling();
@@ -279 +294 @@
 
     private void CreateResultPollingThreads() {
-      foreach(JobItem jobItem in JobItems) {
+      foreach (JobItem jobItem in JobItems) {
         if (!resultPollingThreads.ContainsKey(jobItem.JobDto.Id)) {
           resultPollingThreads.Add(jobItem.JobDto.Id, CreateResultPollingThread(jobItem.JobDto));
@@ -314 +329 @@
     /// <param name="flatout">if false only top level optimizers are returned, if true the optimizer-tree is flatted</param>
     /// <returns></returns>
-    private IEnumerable<IOptimizer> GetOptimizers(bool flatout) {
+    private IDictionary<IOptimizer, IOptimizer> GetOptimizers(bool flatout) {
       if (!flatout) {
-        return experiment.Optimizers;
+        var optimizers = new Dictionary<IOptimizer, IOptimizer>();
+        foreach (IOptimizer opt in experiment.Optimizers) {
+          optimizers.Add(experiment, opt);
+        }
+        return optimizers;
       } else {
-        throw new NotImplementedException();
-      }
-    }
-
-    private void ReplaceOptimizer(IOptimizer originalOptimizer, IOptimizer newOptimizer) {
+        return FlatOptimizerTree(null, experiment);
+      }
+    }
+
+    /// <summary>
+    /// Recursively iterates all IOptimizers in the optimizer-tree and returns them.
+    /// 
+    /// [chn] this could be implemented more cleanly if Experiment and BatchRun would implement an interface like:
+    /// interface IParallelizable {
+    ///   IEnumerable&lt;IOptimizer&gt; GetOptimizers();
+    /// }
+    /// </summary>
+    /// <returns>a dictionary mapping from the parent optimizer to the child optimizer</returns>
+    private IDictionary<IOptimizer, IOptimizer> FlatOptimizerTree(IOptimizer parent, IOptimizer optimizer) {
+      IDictionary<IOptimizer, IOptimizer> optimizers = new Dictionary<IOptimizer, IOptimizer>();
+      if (optimizer is HeuristicLab.Optimization.Experiment) {
+        HeuristicLab.Optimization.Experiment experiment = optimizer as HeuristicLab.Optimization.Experiment;
+        foreach (IOptimizer opt in experiment.Optimizers) {
+          AddRange(optimizers, FlatOptimizerTree(experiment, opt));
+        }
+      } else if (optimizer is BatchRun) {
+        BatchRun batchRun = optimizer as BatchRun;
+        for (int i = 0; i < batchRun.Repetitions; i++) {
+          IOptimizer opt = (IOptimizer)batchRun.Algorithm.Clone();
+          AddRange(optimizers, FlatOptimizerTree(batchRun, opt));
+        }
+      } else if (optimizer is EngineAlgorithm) {
+        optimizers.Add(optimizer, parent);
+      } else {
+        Logger.Warn("Optimizer of type " + optimizers.GetType().ToString() + " unknown");
+        optimizers.Add(optimizer, parent);
+      }
+      return optimizers;
+    }
+
+    private void AddRange(IDictionary<IOptimizer, IOptimizer> optimizers, IDictionary<IOptimizer, IOptimizer> childs) {
+      foreach (KeyValuePair<IOptimizer, IOptimizer> kvp in childs) {
+        optimizers.Add(kvp);
+      }
+    }
+
+    private void ReplaceOptimizer(IOptimizer parentOptimizer, IOptimizer originalOptimizer, IOptimizer newOptimizer) {
       lock (locker) {
-        int originalOptimizerIndex = experiment.Optimizers.IndexOf(originalOptimizer);
-        experiment.Optimizers[originalOptimizerIndex] = newOptimizer;
+        if (parentOptimizer is HeuristicLab.Optimization.Experiment) {
+          HeuristicLab.Optimization.Experiment exp = (HeuristicLab.Optimization.Experiment)parentOptimizer;
+          int originalOptimizerIndex = exp.Optimizers.IndexOf(originalOptimizer);
+          exp.Optimizers[originalOptimizerIndex] = newOptimizer;
+        } else if (parentOptimizer is BatchRun) {
+          BatchRun batchRun = (BatchRun)parentOptimizer;
+          if (newOptimizer is IAlgorithm) {
+            batchRun.Runs.Add(new Run((IAlgorithm)newOptimizer));
+          } else {
+            throw new NotSupportedException("Only IAlgorithm types supported");
+          }
+        } else {
+          throw new NotSupportedException("Invalid parentOptimizer");
+        }
       }
     }
@@ -399 +467 @@
               GetJobItemById(job.Id).LogMessage("Job successfully aborted");
               return;
-            }            
+            }
 
             // loop while 
@@ -416 +484 @@
           GetJobItemById(job.Id).LogMessage("Job finished");
           // job retrieved... replace the existing optimizers with the finished one
-          IOptimizer originalOptimizer = pendingOptimizers[job.Id];
+          IOptimizer originalOptimizer = pendingOptimizersByJobId[job.Id];
           IOptimizer restoredOptimizer = ((OptimizerJob)restoredObject).Optimizer;
 
-          ReplaceOptimizer(originalOptimizer, restoredOptimizer);
-          pendingOptimizers.Remove(job.Id);
-
-          if (pendingOptimizers.Count == 0) {
+          ReplaceOptimizer(parentOptimizersByPendingOptimizer[originalOptimizer], originalOptimizer, restoredOptimizer);
+          pendingOptimizersByJobId.Remove(job.Id);
+          parentOptimizersByPendingOptimizer.Remove(originalOptimizer);
+
+          if (pendingOptimizersByJobId.Count == 0) {
             // finished
             this.ExecutionState = Core.ExecutionState.Stopped;