Free cookie consent management tool by TermsFeed Policy Generator

Changeset 5535


Ignore:
Timestamp:
02/22/11 13:54:31 (14 years ago)
Author:
cneumuel
Message:

#1233

  • merged changes in hive experiment manager from 3.3 (more reliable and efficient downloading of jobs)
Location:
branches/HeuristicLab.Hive-3.4/sources
Files:
1 added
11 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Calendar/3.3

    • Property svn:ignore

      •  

        old new  
        22bin
        33obj
         4*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.ConsoleClient-3.4

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.SlaveTrayIcon

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.SlaveWindowsService

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Tests

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Tests

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs

    r5526 r5535  
    325325      JobData jobData;
    326326      List<IPluginDescription> plugins;
    327      
     327
    328328      if (hiveJob.OptimizerJob.ComputeInParallel &&
    329329        (hiveJob.OptimizerJob.Optimizer is Optimization.Experiment || hiveJob.OptimizerJob.Optimizer is Optimization.BatchRun)) {
    … …  
    599599        IsProgressing = true;
    600600        int totalJobCount = 0;
    601         int jobCount = 0;
     601        IEnumerable<LightweightJob> allJobs;
     602
    602603        progress.Status = "Connecting to Server...";
    603         ServiceLocator.Instance.CallHiveService(service => {
    604           // fetch all Job objects to create the full tree of tree of HiveJob objects
    605           progress.Status = "Downloading list of jobs...";
    606           IEnumerable<LightweightJob> allResults = service.GetLightweightChildJobs(rootJobId, true, true);
    607           totalJobCount = allResults.Count();
    608 
    609           // download them first
    610           IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
    611           IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
    612           foreach (LightweightJob lightweightJob in allResults) {
    613             jobCount++;
    614             progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
    615             allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
    616             allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
    617             progress.ProgressValue = (double)jobCount / totalJobCount;
    618           }
    619 
    620           jobCount = 1;
    621           progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
    622           this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
    623           allJobDatas.Remove(this.rootJobId); // reduce memory footprint
    624           allJobs.Remove(this.rootJobId);
    625           progress.ProgressValue = (double)jobCount / totalJobCount;
    626 
    627           if (this.HiveJob.Job.DateFinished.HasValue) {
    628             this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
    629             this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
    630             this.ExecutionState = Core.ExecutionState.Stopped;
    631             OnStopped();
    632           } else {
    633             this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
    634             this.lastUpdateTime = DateTime.Now;
    635             this.ExecutionState = Core.ExecutionState.Started;
    636             OnStarted();
    637           }
    638 
    639           // build child-job tree
    640           LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
    641           StartResultPolling();
    642         });
     604        // fetch all Job objects to create the full tree of tree of HiveJob objects
     605        progress.Status = "Downloading list of jobs...";
     606        allJobs = ServiceLocator.Instance.CallHiveService(s => s.GetLightweightChildJobs(rootJobId, true, true));
     607        totalJobCount = allJobs.Count();
     608
     609        HiveJobDownloader downloader = new HiveJobDownloader(allJobs.Select(x => x.Id));
     610        downloader.StartAsync();
     611
     612        while (!downloader.IsFinished) {
     613          progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount;
     614          progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
     615          Thread.Sleep(500);
     616        }
     617        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
     618
     619        this.HiveJob = allHiveJobs[this.rootJobId];
     620
     621
     622        //// download them first
     623        //IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
     624        //IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
     625        //foreach (LightweightJob lightweightJob in allResults) {
     626        //  jobCount++;
     627        //  progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
     628        //  allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
     629        //  allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
     630        //  progress.ProgressValue = (double)jobCount / totalJobCount;
     631        //}
     632
     633        //jobCount = 1;
     634        //progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
     635        //this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
     636        //allJobDatas.Remove(this.rootJobId); // reduce memory footprint
     637        //allJobs.Remove(this.rootJobId);
     638        //progress.ProgressValue = (double)jobCount / totalJobCount;
     639
     640        if (this.HiveJob.Job.DateFinished.HasValue) {
     641          this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
     642          this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
     643          this.ExecutionState = Core.ExecutionState.Stopped;
     644          OnStopped();
     645        } else {
     646          this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
     647          this.lastUpdateTime = DateTime.Now;
     648          this.ExecutionState = Core.ExecutionState.Started;
     649          OnStarted();
     650        }
     651
     652        // build child-job tree
     653        //LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
     654        BuildHiveJobTree(this.HiveJob, allJobs, allHiveJobs);
     655        StartResultPolling();
    643656      }
    644657      catch (Exception e) {
    … …  
    650663    }
    651664
    652     private void LoadChildResults(IHiveService service, HiveJob parentHiveJob, IEnumerable<LightweightJob> allResults, IDictionary<Guid, Job> allJobs, IDictionary<Guid, JobData> allJobDatas, IProgress progress, int totalJobCount, ref int jobCount) {
    653       IEnumerable<LightweightJob> childResults = from result in allResults
    654                                                  where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.Job.Id
    655                                                  orderby result.DateCreated ascending
    656                                                  select result;
    657       foreach (LightweightJob lightweightJob in childResults) {
    658         jobCount++;
    659         progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allJobDatas[lightweightJob.Id].Data.Count() / 1024);
    660         OptimizerJob optimizerJob = null;
    661         try {
    662           optimizerJob = PersistenceUtil.Deserialize<OptimizerJob>(allJobDatas[lightweightJob.Id].Data);
    663         }
    664         catch {
    665           optimizerJob = null;
    666         }
    667         progress.ProgressValue = (double)jobCount / totalJobCount;
    668         HiveJob childHiveJob = new HiveJob(optimizerJob, false);
     665    private void BuildHiveJobTree(HiveJob parentHiveJob, IEnumerable<LightweightJob> allJobs, IDictionary<Guid, HiveJob> allHiveJobs) {
     666      IEnumerable<LightweightJob> childJobs = from job in allJobs
     667                                              where job.ParentJobId.HasValue && job.ParentJobId.Value == parentHiveJob.Job.Id
     668                                              orderby job.DateCreated ascending
     669                                              select job;
     670      foreach (LightweightJob job in childJobs) {
     671        HiveJob childHiveJob = allHiveJobs[job.Id];
    669672        parentHiveJob.AddChildHiveJob(childHiveJob);
    670         childHiveJob.Job = allJobs[lightweightJob.Id];
    671         allJobDatas.Remove(lightweightJob.Id); // reduce memory footprint
    672         allJobs.Remove(lightweightJob.Id);
    673         if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high
    674         LoadChildResults(service, childHiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
    675       }
    676     }
    677 
     673        BuildHiveJobTree(childHiveJob, allJobs, allHiveJobs);
     674      }
     675    }
     676   
    678677    private OptimizerJob LoadOptimizerJob(Guid jobId) {
    679       return ServiceLocator.Instance.CallHiveService(service => {
    680         JobData jobData = service.GetJobData(jobId);
    681         try {
    682           return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
    683         }
    684         catch {
    685           return null;
    686         }
    687       });
     678      JobData jobData = ServiceLocator.Instance.CallHiveService(s => s.GetJobData(jobId));
     679      try {
     680        return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
     681      }
     682      catch {
     683        return null;
     684      }
    688685    }
    689686    #endregion

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobClient.cs

    r5526 r5535  
    114114    }
    115115
    116     //public HiveJob(Job jobDto)
    117     //  : this() {
    118     //  this.Job = jobDto;
    119     //}
    120 
    121     //public HiveJob(LightweightJob lightweightJob)
    122     //  : this() {
    123     //  UpdateFromLightweightJob(lightweightJob);
    124     //}
    125 
    126116    public HiveJob(OptimizerJob optimizerJob, bool autoCreateChildHiveJobs)
    127117      : this() {
    … …  
    145135        this.OptimizerJob = null;
    146136      }
     137      this.childHiveJobs = new ItemList<HiveJob>();
    147138      this.syncJobsWithOptimizers = true;
    148139    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/HeuristicLab.Clients.Hive-3.4.csproj

    r5458 r5535  
    118118    <Compile Include="Exceptions\JobResultPollingException.cs" />
    119119    <Compile Include="Exceptions\OptimizerNotFoundException.cs" />
     120    <Compile Include="ExperimentManager\HiveJobDownloader.cs" />
    120121    <Compile Include="ExperimentManager\PluginClient.cs" />
    121122    <Compile Include="Exceptions\ResourceNotFoundException.cs" />

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests

    • Property svn:ignore

      •  

        old new  
        11bin
        22obj
         3*.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HiveService.cs

    r5526 r5535  
    388388      var jobs = new List<Job>(dao.GetJobs(x => parentJobId == null ? !x.ParentJobId.HasValue : x.ParentJobId.Value == parentJobId));
    389389
    390       if (includeParent) {
    391         jobs.Add(GetJob(parentJobId.Value));
    392       }
    393 
    394390      if (recursive) {
    395391        var childs = new List<Job>();
    … …  
    399395        jobs.AddRange(childs);
    400396      }
     397
     398      if (includeParent) jobs.Add(GetJob(parentJobId.Value));
     399
    401400      return jobs;
    402401    }
Note: See TracChangeset for help on using the changeset viewer.