Changeset 5179 for branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager

Timestamp:

12/29/10 00:52:06 (14 years ago)

Author:

cneumuel

Message:

#1260

• migrated to .NET 4.0
• moved state-information about heartbeat timestamps into DB to reduce IIS-recycling issues
• optimized memory usage of ExperimentManager when lots of large jobs are downloaded

Location:

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3

Files:

• HeuristicLab.Hive.ExperimentManager-3.3.csproj (modified) (4 diffs)
• HiveExperiment.cs (modified) (4 diffs)
• HiveJobDownloader.cs (added)
• Jobs/OptimizerJob.cs (modified) (2 diffs)
• Properties/AssemblyInfo.cs.frame (modified) (1 diff)
• app.config (modified) (1 diff)
• christoph - app.config (added)

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HeuristicLab.Hive.ExperimentManager-3.3.csproj

@@ -11 +11 @@
     <RootNamespace>HeuristicLab.Hive.ExperimentManager</RootNamespace>
     <AssemblyName>HeuristicLab.Hive.ExperimentManager-3.3</AssemblyName>
-    <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <TargetFrameworkProfile>
     </TargetFrameworkProfile>
@@ -81 +81 @@
   <ItemGroup>
     <Reference Include="HeuristicLab.Clients.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
-      <HintPath>..\..\..\..\..\..\..\..\Program Files\HeuristicLab 3.3\HeuristicLab.Clients.Common-3.3.dll</HintPath>
+      <HintPath>C:\Program Files\HeuristicLab 3.3\HeuristicLab.Clients.Common-3.3.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Collections-3.3">
@@ -121 +121 @@
       <SubType>Designer</SubType>
     </None>
+    <None Include="christoph - app.config">
+      <SubType>Designer</SubType>
+    </None>
     <None Include="f005pc.hagenberg.fhooe.at - app.config">
       <SubType>Designer</SubType>
@@ -126 +129 @@
     <None Include="HeuristicLabHiveExperimentManagerPlugin.cs.frame" />
     <Compile Include="Exceptions\AddJobToHiveException.cs" />
+    <Compile Include="HiveJobDownloader.cs" />
     <Compile Include="HiveExperiment.cs" />
     <Compile Include="HiveJob.cs" />

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveExperiment.cs

@@ -534 +534 @@
         IsProgressing = true;
         int totalJobCount = 0;
-        int jobCount = 0;
         JobResultList allResults;
-        IDictionary<Guid, SerializedJob> allSerializedJobs = new Dictionary<Guid, SerializedJob>();
+
         progress.Status = "Connecting to Server...";
-        using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
-          // fetch all JobDto objects to create the full tree of tree of HiveJob objects
+        using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
           progress.Status = "Downloading list of jobs...";
           allResults = service.Obj.GetChildJobResults(rootJobId.Value, true, true).Obj;
           totalJobCount = allResults.Count;
-
-          // download them first
-          foreach (JobResult jobResult in allResults) {
-            jobCount++;
-            progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
-            allSerializedJobs.Add(jobResult.Id, service.Obj.GetLastSerializedResult(jobResult.Id).Obj);
-            progress.ProgressValue = (double)jobCount / totalJobCount;
-          }
-        }
-
-        jobCount = 1;
-        progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allSerializedJobs[this.rootJobId.Value].SerializedJobData.Count() / 1024);
-        this.HiveJob = new HiveJob(allSerializedJobs[this.rootJobId.Value], false);
-        allSerializedJobs.Remove(this.rootJobId.Value); // reduce memory footprint
-        progress.ProgressValue = (double)jobCount / totalJobCount;
+        }
+
+        HiveJobDownloader downloader = new HiveJobDownloader(allResults.Select(x => x.Id));
+        downloader.StartAsync();
+
+        while (!downloader.IsFinished) {
+          progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount;
+          progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
+          Thread.Sleep(500);
+        }
+        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
+
+        this.HiveJob = allHiveJobs[this.rootJobId.Value];
 
         if (this.HiveJob.JobDto.DateFinished.HasValue) {
@@ -571 +567 @@
         }
 
-        // build child-job tree
-        LoadChildResults(this.HiveJob, allResults, allSerializedJobs, progress, totalJobCount, ref jobCount);
+        BuildHiveJobTree(this.HiveJob, allResults, allHiveJobs);
         StartResultPolling();
       }
@@ -583 +578 @@
     }
 
-    private void LoadChildResults(HiveJob parentHiveJob, JobResultList allResults, IDictionary<Guid, SerializedJob> allSerializedJobs, IProgress progress, int totalJobCount, ref int jobCount) {
+    private void BuildHiveJobTree(HiveJob parentHiveJob, JobResultList allResults, IDictionary<Guid, HiveJob> allHiveJobs) {
       IEnumerable<JobResult> childResults = from result in allResults
                                             where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.JobDto.Id
@@ -589 +584 @@
                                             select result;
       foreach (JobResult jobResult in childResults) {
-        jobCount++;
-        progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allSerializedJobs[jobResult.Id].SerializedJobData.Count() / 1024);
-        OptimizerJob optimizerJob = null;
-        try {
-          optimizerJob = SerializedJob.Deserialize<OptimizerJob>(allSerializedJobs[jobResult.Id].SerializedJobData);
-        }
-        catch {
-          optimizerJob = null;
-        }
-        progress.ProgressValue = (double)jobCount / totalJobCount;
-        HiveJob childHiveJob = new HiveJob(optimizerJob, false);
+        HiveJob childHiveJob = allHiveJobs[jobResult.Id];
         parentHiveJob.AddChildHiveJob(childHiveJob);
-        childHiveJob.JobDto = allSerializedJobs[jobResult.Id].JobInfo;
-        allSerializedJobs.Remove(jobResult.Id); // reduce memory footprint
-        if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high
-        LoadChildResults(childHiveJob, allResults, allSerializedJobs, progress, totalJobCount, ref jobCount);
+        BuildHiveJobTree(childHiveJob, allResults, allHiveJobs);
       }
     }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/Jobs/OptimizerJob.cs

@@ -108 +108 @@
       return new OptimizerJob(this, cloner);
     }
-    public object Clone() {
-      return Clone(new Cloner());
-    }
     [StorableHook(HookType.AfterDeserialization)]
     protected virtual void AfterDeserialization() {
@@ -256 +253 @@
 
     protected virtual void optimizer_Stopped(object sender, EventArgs e) {
+      optimizer.Prepare(); // reduce memory consumption
       OnJobStopped();
     }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/Properties/AssemblyInfo.cs.frame

@@ -36 +36 @@
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
-[assembly: AllowPartiallyTrustedCallers]
 
 // Setting ComVisible to false makes the types in this assembly not visible 

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/app.config

@@ -29 +29 @@
     </bindings>
     <client>
-      <endpoint address="net.tcp://services.heuristiclab.com:8000/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
+      <endpoint address="net.tcp://christoph:9001/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
         <identity>
-          <dns value="services.heuristiclab.com"/>
+          <dns value="localhost"/>
         </identity>
       </endpoint>
-      <endpoint address="http://services.heuristiclab.com/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
+      <endpoint address="http://christoph/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
         </identity>
       </endpoint>
     </client>
-    
+
   </system.serviceModel>
 <startup><supportedRuntime version="v2.0.50727"/></startup></configuration>