Free cookie consent management tool by TermsFeed Policy Generator

Changeset 6219


Ignore:
Timestamp:
05/17/11 11:34:40 (13 years ago)
Author:
cneumuel
Message:

#1233

  • improved exception handling for hive experiments
Location:
branches/HeuristicLab.Hive-3.4/sources
Files:
13 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/RefreshableHiveExperimentView.cs

    r6200 r6219  
    6060      Content.IsControllableChanged += new EventHandler(Content_IsControllableChanged);
    6161      Content.JobStatisticsChanged += new EventHandler(Content_JobStatisticsChanged);
     62      Content.ExceptionOccured += new EventHandler<EventArgs<Exception>>(Content_ExceptionOccured);
    6263    }
    6364
    … …  
    6768      Content.IsControllableChanged -= new EventHandler(Content_IsControllableChanged);
    6869      Content.JobStatisticsChanged -= new EventHandler(Content_JobStatisticsChanged);
     70      Content.ExceptionOccured -= new EventHandler<EventArgs<Exception>>(Content_ExceptionOccured);
    6971      base.DeregisterContentEvents();
    7072    }
    … …  
    252254      }
    253255    }
     256    private void Content_ExceptionOccured(object sender, EventArgs<Exception> e) {
     257      if (Content.IsControllable) {
     258        // show error dialog only if Controllable (otherwise it should continue trying without an error dialog (e.g. HiveEngine))
     259        ErrorHandling.ShowErrorDialog(this, e.Value);
     260      }
     261    }
    254262    #endregion
    255263

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ConcurrentJobDownloader.cs

    r6200 r6219  
    22using System.Threading;
    33using System.Threading.Tasks;
     4using HeuristicLab.Common;
    45using HeuristicLab.Hive;
    56
    … …  
    2021    }
    2122
    22     public void DownloadJob(Job job, Action<Job, T, Exception> onFinishedAction) {
     23    public void DownloadJob(Job job, Action<Job, T> onFinishedAction) {
    2324      Task<T> task = Task<JobData>.Factory.StartNew((x) => DownloadJob(x), job.Id)
    2425                                     .ContinueWith((x) => DeserializeJob(x.Result));
    … …  
    2728    }
    2829
    29     private void OnJobFinished(Job job, Task<T> task, Action<Job, T, Exception> onFinishedAction) {
    30       onFinishedAction(job, task.Result, null);
     30    private void OnJobFinished(Job job, Task<T> task, Action<Job, T> onFinishedAction) {
     31      onFinishedAction(job, task.Result);
    3132    }
    32     private void OnJobFailed(Job job, Task<T> task, Action<Job, T, Exception> onFinishedAction) {
    33       onFinishedAction(job, task.Result, task.Exception.Flatten());
     33    private void OnJobFailed(Job job, Task<T> task, Action<Job, T> onFinishedAction) {
     34      task.Exception.Flatten().Handle((e) => { return true; });
     35      OnExceptionOccured(task.Exception.Flatten());
     36      onFinishedAction(job, null);
    3437    }
    3538
    … …  
    6467    private void TaskScheduler_UnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) {
    6568      e.SetObserved(); // avoid crash of process because task crashes. first exception found is handled in Results property
     69      OnExceptionOccured(new HiveException("Unobserved Exception in ConcurrentJobDownloader", e.Exception));
     70    }
     71
     72    public event EventHandler<EventArgs<Exception>> ExceptionOccured;
     73    private void OnExceptionOccured(Exception exception) {
     74      var handler = ExceptionOccured;
     75      if (handler != null) handler(this, new EventArgs<Exception>(exception));
    6676    }
    6777  }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ExperimentManagerClient.cs

    r6200 r6219  
    192192
    193193    #region Upload Experiment
    194     private void UploadExperiment(RefreshableHiveExperiment optimizerHiveExperiment) {
     194    private void UploadExperiment(RefreshableHiveExperiment refreshableHiveExperiment) {
    195195      try {
    196         optimizerHiveExperiment.HiveExperiment.Progress = new Progress("Connecting to server...");
    197         optimizerHiveExperiment.HiveExperiment.IsProgressing = true;
     196        refreshableHiveExperiment.HiveExperiment.Progress = new Progress("Connecting to server...");
     197        refreshableHiveExperiment.HiveExperiment.IsProgressing = true;
    198198        ServiceLocator.Instance.CallHiveService(service => {
    199           IEnumerable<string> resourceNames = ToResourceNameList(optimizerHiveExperiment.HiveExperiment.ResourceNames);
     199          IEnumerable<string> resourceNames = ToResourceNameList(refreshableHiveExperiment.HiveExperiment.ResourceNames);
    200200          var resourceIds = new List<Guid>();
    201201          foreach (var resourceName in resourceNames) {
    … …  
    207207          }
    208208
    209           foreach (OptimizerHiveJob hiveJob in optimizerHiveExperiment.HiveExperiment.HiveJobs.OfType<OptimizerHiveJob>()) {
     209          foreach (OptimizerHiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs.OfType<OptimizerHiveJob>()) {
    210210            hiveJob.SetIndexInParentOptimizerList(null);
    211211          }
    212212
    213213          // upload HiveExperiment
    214           optimizerHiveExperiment.HiveExperiment.Progress.Status = "Uploading HiveExperiment...";
    215           optimizerHiveExperiment.HiveExperiment.Id = service.AddHiveExperiment(optimizerHiveExperiment.HiveExperiment);
    216 
    217           int totalJobCount = optimizerHiveExperiment.HiveExperiment.GetAllHiveJobs().Count();
     214          refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading HiveExperiment...";
     215          refreshableHiveExperiment.HiveExperiment.Id = service.AddHiveExperiment(refreshableHiveExperiment.HiveExperiment);
     216
     217          int totalJobCount = refreshableHiveExperiment.HiveExperiment.GetAllHiveJobs().Count();
    218218          int jobCount = 0;
    219219
    220220          // upload plugins
    221           optimizerHiveExperiment.HiveExperiment.Progress.Status = "Uploading plugins...";
     221          refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading plugins...";
    222222          this.OnlinePlugins = service.GetPlugins();
    223223          this.AlreadyUploadedPlugins = new List<Plugin>();
    … …  
    226226
    227227          // upload jobs
    228           optimizerHiveExperiment.HiveExperiment.Progress.Status = "Uploading jobs...";
    229 
    230           foreach (HiveJob hiveJob in optimizerHiveExperiment.HiveExperiment.HiveJobs) {
    231             UploadJobWithChildren(optimizerHiveExperiment.HiveExperiment.Progress, service, hiveJob, null, resourceIds, ref jobCount, totalJobCount, configFilePlugin.Id, optimizerHiveExperiment.HiveExperiment.UseLocalPlugins, optimizerHiveExperiment.HiveExperiment.Id);
     228          refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading jobs...";
     229
     230          foreach (HiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs) {
     231            UploadJobWithChildren(refreshableHiveExperiment.HiveExperiment.Progress, service, hiveJob, null, resourceIds, ref jobCount, totalJobCount, configFilePlugin.Id, refreshableHiveExperiment.HiveExperiment.UseLocalPlugins, refreshableHiveExperiment.HiveExperiment.Id, refreshableHiveExperiment.Log);
    232232          }
    233233
    234           if (optimizerHiveExperiment.RefreshAutomatically) optimizerHiveExperiment.StartResultPolling();
     234          if (refreshableHiveExperiment.RefreshAutomatically) refreshableHiveExperiment.StartResultPolling();
    235235        });
    236236      }
    237237      finally {
    238         optimizerHiveExperiment.HiveExperiment.IsProgressing = false;
     238        refreshableHiveExperiment.HiveExperiment.IsProgressing = false;
    239239      }
    240240    }
    … …  
    268268    /// <param name="parentHiveJob">shall be null if its the root job</param>
    269269    /// <param name="groups"></param>
    270     private void UploadJobWithChildren(IProgress progress, IHiveService service, HiveJob hiveJob, HiveJob parentHiveJob, IEnumerable<Guid> groups, ref int jobCount, int totalJobCount, Guid configPluginId, bool useLocalPlugins, Guid hiveExperimentId) {
     270    private void UploadJobWithChildren(IProgress progress, IHiveService service, HiveJob hiveJob, HiveJob parentHiveJob, IEnumerable<Guid> groups, ref int jobCount, int totalJobCount, Guid configPluginId, bool useLocalPlugins, Guid hiveExperimentId, ILog log) {
    271271      jobCount++;
    272272      progress.Status = string.Format("Serializing job {0} of {1}", jobCount, totalJobCount);
    … …  
    292292      hiveJob.Job.HiveExperimentId = hiveExperimentId;
    293293
    294       progress.Status = string.Format("Uploading job {0} of {1} ({2} kb)", jobCount, totalJobCount, jobData.Data.Count() / 1024);
     294      progress.Status = string.Format("Uploading job {0} of {1} ({2} kb, {3} objects)", jobCount, totalJobCount, jobData.Data.Count() / 1024, hiveJob.ItemJob.GetObjectGraphObjects().Count());
    295295      progress.ProgressValue = (double)jobCount / totalJobCount;
    296296
    297 
     297      log.LogMessage(progress.Status);
    298298      TryAndRepeat(() => {
    299299        if (parentHiveJob != null) {
    … …  
    305305
    306306      foreach (HiveJob child in hiveJob.ChildHiveJobs) {
    307         UploadJobWithChildren(progress, service, child, hiveJob, groups, ref jobCount, totalJobCount, configPluginId, useLocalPlugins, hiveExperimentId);
     307        UploadJobWithChildren(progress, service, child, hiveJob, groups, ref jobCount, totalJobCount, configPluginId, useLocalPlugins, hiveExperimentId, log);
    308308      }
    309309    }
    … …  
    331331          hiveExperiment.Progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
    332332          Thread.Sleep(500);
     333
     334          if (downloader.IsFaulted) {
     335            throw downloader.Exception;
     336          }
    333337        }
    334338        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
    … …  
    393397    /// </summary>
    394398    public static void TryAndRepeat(Action action, int repetitions, string errorMessage) {
    395       try { action(); }
    396       catch (Exception e) {
    397         repetitions--;
    398         if (repetitions == 0)
    399           throw new HiveException(errorMessage, e);
    400         TryAndRepeat(action, repetitions, errorMessage);
     399      while (true) {
     400        try { action(); return; }
     401        catch (Exception e) {
     402          if (repetitions == 0)
     403            throw new HiveException(errorMessage, e);
     404          repetitions--;
     405        }
    401406      }
    402407    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobDownloader.cs

    r6198 r6219  
    2525using HeuristicLab.Clients.Hive.ExperimentManager;
    2626using HeuristicLab.Clients.Hive.Jobs;
     27using HeuristicLab.Common;
    2728using HeuristicLab.Hive;
    2829
    … …  
    3233    private ConcurrentJobDownloader<ItemJob> jobDownloader;
    3334    private IDictionary<Guid, HiveJob> results;
     35    private bool exceptionOccured = false;
     36    private Exception currentException;
    3437
    3538    public bool IsFinished {
    … …  
    3942    }
    4043
     44    public bool IsFaulted {
     45      get {
     46        return exceptionOccured;
     47      }
     48    }
     49
     50    public Exception Exception {
     51      get {
     52        return currentException;
     53      }
     54    }
     55   
    4156    public int FinishedCount {
    4257      get {
    … …  
    5469      this.jobIds = jobIds;
    5570      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     71      this.jobDownloader.ExceptionOccured += new EventHandler<EventArgs<Exception>>(jobDownloader_ExceptionOccured);
    5672      this.results = new Dictionary<Guid, HiveJob>();
    5773    }
    58 
     74   
    5975    public void StartAsync() {
    6076      foreach (Guid jobId in jobIds) {
    … …  
    6278
    6379        jobDownloader.DownloadJob(job,
    64           (localJob, itemJob, exception) => {
    65             if (exception != null) {
    66               throw new ConcurrentJobDownloaderException("Downloading job failed", exception);
    67             }
     80          (localJob, itemJob) => {
    6881            if (localJob != null && itemJob != null) {
    6982              HiveJob hiveJob;
    … …  
    7992      }     
    8093    }
     94
     95    private void jobDownloader_ExceptionOccured(object sender, EventArgs<Exception> e) {
     96      OnExceptionOccured(e.Value);
     97    }
     98
     99    public event EventHandler<EventArgs<Exception>> ExceptionOccured;
     100    private void OnExceptionOccured(Exception exception) {
     101      this.exceptionOccured = true;
     102      this.currentException = exception;
     103      var handler = ExceptionOccured;
     104      if (handler != null) handler(this, new EventArgs<Exception>(exception));
     105    }
    81106  }
    82107}

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/JobResultPoller.cs

    r6111 r6219  
    5454    }
    5555
     56    private bool autoResumeOnException = false;
     57    public bool AutoResumeOnException {
     58      get { return autoResumeOnException; }
     59      set { autoResumeOnException = value; }
     60    }
     61
     62
    5663    public JobResultPoller(Guid hiveExperimentId, TimeSpan interval) {
    5764      this.isPolling = false;
    … …  
    6168
    6269    public void Start() {
     70      IsPolling = true;
    6371      stopRequested = false;
    6472      thread = new Thread(RunPolling);
    6573      thread.Start();
    66       IsPolling = true;
    6774    }
    6875
    6976    public void Stop() {
    70       // use AutoResetEvent.Set instead if Thread.Interrupt because its much cleaner
    7177      stopRequested = true;
    7278      waitHandle.Set();
    … …  
    7682
    7783    private void RunPolling() {
    78       try {
    79         waitHandle = new AutoResetEvent(false);
    80         while (!stopRequested) {
    81           OnPollingStarted();
    82           FetchJobResults();
    83           OnPollingFinished();
    84           waitHandle.WaitOne(Interval);
     84      while (true) {
     85        IsPolling = true;
     86        try {
     87          waitHandle = new AutoResetEvent(false);
     88          while (!stopRequested) {
     89            OnPollingStarted();
     90            FetchJobResults();
     91            OnPollingFinished();
     92            waitHandle.WaitOne(Interval);
     93          }
     94          waitHandle.Close();
    8595        }
    86         waitHandle.Close();
    87       }
    88       catch (Exception e) {
    89         OnExceptionOccured(e);
    90       }
    91       finally {
    92         IsPolling = false;
     96        catch (Exception e) {
     97          OnExceptionOccured(e);
     98        }
     99        finally {
     100          IsPolling = false;
     101        }
     102        if (!autoResumeOnException) return;
    93103      }
    94104    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/RefreshableHiveExperiment.cs

    r6200 r6219  
    3232  public class RefreshableHiveExperiment : IHiveItem, IDeepCloneable, IContent, IProgressReporter {
    3333    private JobResultPoller jobResultPoller;
    34     private ConcurrentJobDownloader<ItemJob> jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     34    private ConcurrentJobDownloader<ItemJob> jobDownloader;
    3535    private static object locker = new object();
    3636
    … …  
    7878    }
    7979
     80    // if true, all control buttons should be enabled. otherwise disabled (used for HiveEngine)
    8081    private bool isControllable = true;
    8182    public bool IsControllable {
    … …  
    103104      this.HiveExperiment = new HiveExperiment();
    104105      this.log = new Log();
     106      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     107      this.jobDownloader.ExceptionOccured += new EventHandler<EventArgs<Exception>>(jobDownloader_ExceptionOccured);
    105108    }
    106109    public RefreshableHiveExperiment(HiveExperiment hiveExperiment) {
    … …  
    109112      this.HiveExperiment = hiveExperiment;
    110113      this.log = new Log();
     114      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     115      this.jobDownloader.ExceptionOccured += new EventHandler<EventArgs<Exception>>(jobDownloader_ExceptionOccured);
    111116    }
    112117    protected RefreshableHiveExperiment(RefreshableHiveExperiment original, Cloner cloner) {
    … …  
    117122      this.Log = cloner.Clone(original.Log);
    118123      this.RefreshAutomatically = false; // do not start results polling automatically
     124      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     125      this.jobDownloader.ExceptionOccured += new EventHandler<EventArgs<Exception>>(jobDownloader_ExceptionOccured);
    119126    }
    120127    public IDeepCloneable Clone(Cloner cloner) {
    … …  
    143150        jobResultPoller = new JobResultPoller(hiveExperiment.Id, /*ApplicationConstants.ResultPollingInterval*/new TimeSpan(0, 0, 5)); //TODO: find a better place for ApplicationConstants
    144151        RegisterResultPollingEvents();
     152        jobResultPoller.AutoResumeOnException = !IsControllable;
    145153      }
    146154
    … …  
    184192            LogMessage(string.Format("Downloading job {0}", lightweightJob.Id));
    185193            hiveJob.IsDownloading = true;
    186             jobDownloader.DownloadJob(hiveJob.Job, (localJob, itemJob, exception) => {
     194            jobDownloader.DownloadJob(hiveJob.Job, (localJob, itemJob) => {
    187195              LogMessage(string.Format("Finished downloading job {0}", localJob.Id));
    188196              HiveJob localHiveJob = GetHiveJobById(localJob.Id);
    189197
    190               if (exception != null) {
    191                 var ex = new ConcurrentJobDownloaderException("Downloading job failed", exception);
    192                 LogException(ex);
     198              if (itemJob == null) {
    193199                localHiveJob.IsDownloading = false;
    194                 throw ex;
    195200              }
    196201
    … …  
    247252      return null;
    248253    }
    249 
    250254    private void UpdateStatistics() {
    251255      var jobs = hiveExperiment.GetAllHiveJobs();
    … …  
    264268
    265269    private void jobResultPoller_ExceptionOccured(object sender, EventArgs<Exception> e) {
    266       //OnExceptionOccured(e.Value);
    267     }
    268 
     270      OnExceptionOccured(e.Value);
     271    }
     272    private void jobDownloader_ExceptionOccured(object sender, EventArgs<Exception> e) {
     273      OnExceptionOccured(e.Value);
     274    }
    269275    public void UpdateTotalExecutionTime() {
    270276      hiveExperiment.ExecutionTime = TimeSpan.FromMilliseconds(hiveExperiment.GetAllHiveJobs().Sum(x => x.Job.ExecutionTime.HasValue ? x.Job.ExecutionTime.Value.TotalMilliseconds : 0));
    … …  
    354360      if (handler != null) handler(this, EventArgs.Empty);
    355361    }
     362
     363    public event EventHandler<EventArgs<Exception>> ExceptionOccured;
     364    private void OnExceptionOccured(Exception exception) {
     365      LogException(exception);
     366      var handler = ExceptionOccured;
     367      if (handler != null) handler(this, new EventArgs<Exception>(exception));
     368    }
    356369    #endregion
    357370

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Hive/3.4/ItemJob.cs

    r6110 r6219  
    4747        }
    4848      }
    49     } 
     49    }
    5050
    5151    [Storable]
    … …  
    108108    #region IJob Members
    109109
    110     public abstract ExecutionState ExecutionState{ get; }
    111 
    112     public abstract TimeSpan ExecutionTime{ get; }
     110    public abstract ExecutionState ExecutionState { get; }
     111
     112    public abstract TimeSpan ExecutionTime { get; }
    113113
    114114    public abstract void Prepare();
    … …  
    176176    }
    177177    #endregion
    178    
     178
    179179    #region INamedItem Members
    180180    public abstract bool CanChangeDescription { get; }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Test

    • Property svn:ignore

      •  

        old new  
        22obj
        33HeuristicLab.HiveEngine.Test.csproj.vs10x
         4HeuristicLab.HiveEngine.Test.csproj.user

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Test/Program.cs

    r6178 r6219  
    3232    public override void Run() {
    3333      ContentManager.Initialize(new PersistenceContentManager());
    34 
    3534
    3635      OptimizerJob job = new OptimizerJob(new Experiment());

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Views/3.4/Properties/AssemblyInfo.cs.frame

    r6217 r6219  
    5454// by using the '*' as shown below:
    5555// [assembly: AssemblyVersion("1.0.*")]
    56 [assembly: AssemblyVersion("3.3.0.0")]
    57 [assembly: AssemblyFileVersion("3.3.0.$WCREV$")]
     56[assembly: AssemblyVersion("3.4.0.0")]
     57[assembly: AssemblyFileVersion("3.4.0.$WCREV$")]

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine/3.4/HeuristicLab.HiveEngine-3.4.csproj

    r6212 r6219  
    134134    <Compile Include="HeuristicLabHiveEnginePlugin.cs" />
    135135    <Compile Include="Exceptions\ScopeMergeException.cs" />
     136    <Compile Include="Properties\AssemblyInfo.cs" />
    136137    <None Include="HeuristicLabHiveEnginePlugin.cs.frame" />
    137138    <Compile Include="HiveEngine.cs" />

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine/3.4/HiveEngine.cs

    r6212 r6219  
    106106      this.executionTimeOnHive = original.executionTimeOnHive;
    107107      this.useLocalPlugins = original.useLocalPlugins;
    108       this.hiveExperiments = cloner.Clone(original.hiveExperiments);
     108      // this.hiveExperiments = cloner.Clone(original.hiveExperiments); do not clone hiveExperiments - otherwise they would be sent with every job
    109109    }
    110110    public override IDeepCloneable Clone(Cloner cloner) {

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine/3.4/Properties/AssemblyInfo.cs.frame

    r6001 r6219  
    5454// by using the '*' as shown below:
    5555// [assembly: AssemblyVersion("1.0.*")]
    56 [assembly: AssemblyVersion("3.3.0.0")]
    57 [assembly: AssemblyFileVersion("3.3.0.$WCREV$")]
     56[assembly: AssemblyVersion("3.4.0.0")]
     57[assembly: AssemblyFileVersion("3.4.0.$WCREV$")]
Note: See TracChangeset for help on using the changeset viewer.