Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
06/19/11 23:21:21 (13 years ago)
Author:
cneumuel
Message:

#1233

  • stability improvements for HiveExperiment and HiveEngine
  • parallelized upload of jobs
  • enabled cancellation of job upload
  • reduced the amount of double-assignment of jobs by an additional check in HeartbeatManager
  • tried to tackle the amount of deadlocks by automatically rerunning transactions
  • some fixes
Location:
branches/HeuristicLab.Hive-3.4/sources
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.ExperimentManager/3.4/Views/RefreshableHiveExperimentView.Designer.cs

    r6426 r6444  
    254254      this.resourceNamesTextBox.Location = new System.Drawing.Point(73, 53);
    255255      this.resourceNamesTextBox.Name = "resourceNamesTextBox";
    256       this.resourceNamesTextBox.Size = new System.Drawing.Size(304, 20);
     256      this.resourceNamesTextBox.Size = new System.Drawing.Size(415, 20);
    257257      this.resourceNamesTextBox.TabIndex = 14;
    258258      this.resourceNamesTextBox.Validated += new System.EventHandler(this.resourceNamesTextBox_Validated);
    … …  
    292292      this.isPrivilegedCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
    293293      this.isPrivilegedCheckBox.AutoSize = true;
    294       this.isPrivilegedCheckBox.Location = new System.Drawing.Point(383, 55);
     294      this.isPrivilegedCheckBox.Location = new System.Drawing.Point(494, 55);
    295295      this.isPrivilegedCheckBox.Name = "isPrivilegedCheckBox";
    296296      this.isPrivilegedCheckBox.Size = new System.Drawing.Size(80, 17);
    … …  
    299299      this.toolTip.SetToolTip(this.isPrivilegedCheckBox, "If checked, the job will be executed in a privileged sandbox on the slave.");
    300300      this.isPrivilegedCheckBox.UseVisualStyleBackColor = true;
    301       this.isPrivilegedCheckBox.CheckedChanged += new System.EventHandler(this.isPrivilegedCheckBox_CheckedChanged);
     301      this.isPrivilegedCheckBox.Validated += new System.EventHandler(this.isPrivilegedCheckBox_Validated);
    302302      //
    303303      // refreshAutomaticallyCheckBox
    … …  
    310310      this.refreshAutomaticallyCheckBox.Text = "&Refresh Automatically";
    311311      this.refreshAutomaticallyCheckBox.UseVisualStyleBackColor = true;
    312       this.refreshAutomaticallyCheckBox.CheckedChanged += new System.EventHandler(this.refreshAutomaticallyCheckBox_CheckedChanged);
     312      this.refreshAutomaticallyCheckBox.Validated += new System.EventHandler(this.refreshAutomaticallyCheckBox_Validated);
    313313      //
    314314      // infoGroupBox

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.ExperimentManager/3.4/Views/RefreshableHiveExperimentView.cs

    r6426 r6444  
    281281    #region Control events
    282282    private void startButton_Click(object sender, EventArgs e) {
    283       HiveClient.StartExperiment((Exception ex) => ErrorHandling.ShowErrorDialog(this, "Start failed.", ex), Content);
     283      HiveClient.StartExperiment((Exception ex) => ErrorHandling.ShowErrorDialog(this, "Start failed.", ex), Content, new CancellationToken());
    284284    }
    285285    private void pauseButton_Click(object sender, EventArgs e) {
    … …  
    301301    }
    302302
    303     private void includeJobsCheckBox_CheckedChanged(object sender, EventArgs e) {
    304       //if (Content != null) Content.IncludeJobs = includeJobsCheckBox.Checked;
    305     }
    306 
    307     private void refreshAutomaticallyCheckBox_CheckedChanged(object sender, EventArgs e) {
    308       if (Content != null) {
    309         Content.RefreshAutomatically = refreshAutomaticallyCheckBox.Checked;
    310       }
    311     }
    312 
    313     private void isPrivilegedCheckBox_CheckedChanged(object sender, EventArgs e) {
     303    private void refreshAutomaticallyCheckBox_Validated(object sender, EventArgs e) {
     304      if (Content != null) Content.RefreshAutomatically = refreshAutomaticallyCheckBox.Checked;
     305    }
     306
     307    private void isPrivilegedCheckBox_Validated(object sender, EventArgs e) {
    314308      if (Content != null) Content.HiveExperiment.IsPrivileged = isPrivilegedCheckBox.Checked;
    315309    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/HiveClient.cs

    r6426 r6444  
    2727using System.Security.Cryptography;
    2828using System.Threading;
     29using System.Threading.Tasks;
    2930using HeuristicLab.Common;
    3031using HeuristicLab.Core;
    … …  
    104105
    105106    #region Store
    106     public static void Store(IHiveItem item) {
     107    public static void Store(IHiveItem item, CancellationToken cancellationToken) {
    107108      if (item.Id == Guid.Empty) {
    108109        if (item is RefreshableHiveExperiment) {
    109           HiveClient.Instance.UploadExperiment((RefreshableHiveExperiment)item);
     110          HiveClient.Instance.UploadExperiment((RefreshableHiveExperiment)item, cancellationToken);
    110111        }
    111112      } else {
    … …  
    114115      }
    115116    }
    116     public static void StoreAsync(Action<Exception> exceptionCallback, IHiveItem item) {
     117    public static void StoreAsync(Action<Exception> exceptionCallback, IHiveItem item, CancellationToken cancellationToken) {
    117118      var call = new Func<Exception>(delegate() {
    118119        try {
    119           Store(item);
     120          Store(item, cancellationToken);
    120121        }
    121122        catch (Exception ex) {
    … …  
    159160    #endregion
    160161
    161     public static void StartExperiment(Action<Exception> exceptionCallback, RefreshableHiveExperiment refreshableHiveExperiment) {
     162    public static void StartExperiment(Action<Exception> exceptionCallback, RefreshableHiveExperiment refreshableHiveExperiment, CancellationToken cancellationToken) {
    162163      HiveClient.StoreAsync(
    163164        new Action<Exception>((Exception ex) => {
    164165          refreshableHiveExperiment.HiveExperiment.ExecutionState = ExecutionState.Prepared;
    165166          exceptionCallback(ex);
    166         }), refreshableHiveExperiment);
     167        }), refreshableHiveExperiment, cancellationToken);
    167168      refreshableHiveExperiment.HiveExperiment.ExecutionState = ExecutionState.Started;
    168169    }
    … …  
    189190
    190191    #region Upload Experiment
    191     private void UploadExperiment(RefreshableHiveExperiment refreshableHiveExperiment) {
     192    private Semaphore jobUploadSemaphore = new Semaphore(4, 4); // todo: take magic number into config
     193    private static object jobCountLocker = new object();
     194    private void UploadExperiment(RefreshableHiveExperiment refreshableHiveExperiment, CancellationToken cancellationToken) {
    192195      try {
    193196        refreshableHiveExperiment.HiveExperiment.Progress = new Progress("Connecting to server...");
    194197        refreshableHiveExperiment.HiveExperiment.IsProgressing = true;
    195         ServiceLocator.Instance.CallHiveService(service => {
    196           IEnumerable<string> resourceNames = ToResourceNameList(refreshableHiveExperiment.HiveExperiment.ResourceNames);
    197           var resourceIds = new List<Guid>();
    198           foreach (var resourceName in resourceNames) {
    199             Guid resourceId = service.GetResourceId(resourceName);
    200             if (resourceId == Guid.Empty) {
    201               throw new ResourceNotFoundException(string.Format("Could not find the resource '{0}'", resourceName));
    202             }
    203             resourceIds.Add(resourceId);
     198
     199        IEnumerable<string> resourceNames = ToResourceNameList(refreshableHiveExperiment.HiveExperiment.ResourceNames);
     200        var resourceIds = new List<Guid>();
     201        foreach (var resourceName in resourceNames) {
     202          Guid resourceId = ServiceLocator.Instance.CallHiveService((s) => s.GetResourceId(resourceName));
     203          if (resourceId == Guid.Empty) {
     204            throw new ResourceNotFoundException(string.Format("Could not find the resource '{0}'", resourceName));
    204205          }
    205 
    206           foreach (OptimizerHiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs.OfType<OptimizerHiveJob>()) {
    207             hiveJob.SetIndexInParentOptimizerList(null);
    208           }
    209 
    210           // upload HiveExperiment
    211           refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading HiveExperiment...";
    212           refreshableHiveExperiment.HiveExperiment.Id = service.AddHiveExperiment(refreshableHiveExperiment.HiveExperiment);
    213 
    214           int totalJobCount = refreshableHiveExperiment.HiveExperiment.GetAllHiveJobs().Count();
    215           int jobCount = 0;
    216 
    217           // upload plugins
    218           refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading plugins...";
    219           this.OnlinePlugins = service.GetPlugins();
    220           this.AlreadyUploadedPlugins = new List<Plugin>();
    221           Plugin configFilePlugin = UploadConfigurationFile(service, onlinePlugins);
    222           this.alreadyUploadedPlugins.Add(configFilePlugin);
    223 
    224           // upload jobs
    225           refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading jobs...";
    226 
    227           foreach (HiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs) {
    228             UploadJobWithChildren(refreshableHiveExperiment.HiveExperiment.Progress, service, hiveJob, null, resourceIds, ref jobCount, totalJobCount, configFilePlugin.Id, refreshableHiveExperiment.HiveExperiment.Id, refreshableHiveExperiment.Log, refreshableHiveExperiment.HiveExperiment.IsPrivileged);
    229           }
    230 
    231           if (refreshableHiveExperiment.RefreshAutomatically) refreshableHiveExperiment.StartResultPolling();
    232         });
     206          resourceIds.Add(resourceId);
     207        }
     208
     209        foreach (OptimizerHiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs.OfType<OptimizerHiveJob>()) {
     210          hiveJob.SetIndexInParentOptimizerList(null);
     211        }
     212
     213        // upload HiveExperiment
     214        refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading HiveExperiment...";
     215        refreshableHiveExperiment.HiveExperiment.Id = ServiceLocator.Instance.CallHiveService((s) => s.AddHiveExperiment(refreshableHiveExperiment.HiveExperiment));
     216        cancellationToken.ThrowIfCancellationRequested();
     217
     218        int totalJobCount = refreshableHiveExperiment.HiveExperiment.GetAllHiveJobs().Count();
     219        int[] jobCount = new int[1]; // use a reference type (int-array) instead of value type (int) in order to pass the value via a delegate to task-parallel-library
     220        cancellationToken.ThrowIfCancellationRequested();
     221
     222        // upload plugins
     223        refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading plugins...";
     224        this.OnlinePlugins = ServiceLocator.Instance.CallHiveService((s) => s.GetPlugins());
     225        this.AlreadyUploadedPlugins = new List<Plugin>();
     226        Plugin configFilePlugin = ServiceLocator.Instance.CallHiveService((s) => UploadConfigurationFile(s, onlinePlugins));
     227        this.alreadyUploadedPlugins.Add(configFilePlugin);
     228        cancellationToken.ThrowIfCancellationRequested();
     229
     230        if (refreshableHiveExperiment.RefreshAutomatically) refreshableHiveExperiment.StartResultPolling();
     231
     232        // upload jobs
     233        refreshableHiveExperiment.HiveExperiment.Progress.Status = "Uploading jobs...";
     234
     235        var tasks = new List<Task>();
     236        foreach (HiveJob hiveJob in refreshableHiveExperiment.HiveExperiment.HiveJobs) {
     237          tasks.Add(Task.Factory.StartNew((hj) => {
     238            UploadJobWithChildren(refreshableHiveExperiment.HiveExperiment.Progress, (HiveJob)hj, null, resourceIds, jobCount, totalJobCount, configFilePlugin.Id, refreshableHiveExperiment.HiveExperiment.Id, refreshableHiveExperiment.Log, refreshableHiveExperiment.HiveExperiment.IsPrivileged, cancellationToken);
     239          }, hiveJob)
     240          .ContinueWith((x) => refreshableHiveExperiment.Log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted));
     241        }
     242        Task.WaitAll(tasks.ToArray());
    233243      }
    234244      finally {
    … …  
    267277    /// Uploads the given job and all its child-jobs while setting the proper parentJobId values for the childs
    268278    /// </summary>
    269     /// <param name="service"></param>
    270     /// <param name="hiveJob"></param>
    271279    /// <param name="parentHiveJob">shall be null if its the root job</param>
    272     /// <param name="groups"></param>
    273     private void UploadJobWithChildren(IProgress progress, IHiveService service, HiveJob hiveJob, HiveJob parentHiveJob, IEnumerable<Guid> groups, ref int jobCount, int totalJobCount, Guid configPluginId, Guid hiveExperimentId, ILog log, bool isPrivileged) {
    274       jobCount++;
    275       progress.Status = string.Format("Serializing job {0} of {1}", jobCount, totalJobCount);
    276       JobData jobData;
    277       List<IPluginDescription> plugins;
    278 
    279       if (hiveJob.ItemJob.ComputeInParallel && (hiveJob.ItemJob.Item is Optimization.Experiment || hiveJob.ItemJob.Item is Optimization.BatchRun)) {
    280         hiveJob.Job.IsParentJob = true;
    281         hiveJob.Job.FinishWhenChildJobsFinished = true;
    282         jobData = hiveJob.GetAsJobData(true, out plugins);
    283       } else {
    284         hiveJob.Job.IsParentJob = false;
    285         hiveJob.Job.FinishWhenChildJobsFinished = false;
    286         jobData = hiveJob.GetAsJobData(false, out plugins);
    287       }
    288 
    289       TryAndRepeat(() => {
    290         hiveJob.Job.PluginsNeededIds = PluginUtil.GetPluginDependencies(service, this.onlinePlugins, this.alreadyUploadedPlugins, plugins);
    291       }, -1, "Failed to upload plugins");
    292       hiveJob.Job.PluginsNeededIds.Add(configPluginId);
    293       hiveJob.Job.HiveExperimentId = hiveExperimentId;
    294       hiveJob.Job.IsPrivileged = isPrivileged;
    295 
    296       progress.Status = string.Format("Uploading job {0} of {1} ({2} kb, {3} objects)", jobCount, totalJobCount, jobData.Data.Count() / 1024, hiveJob.ItemJob.GetObjectGraphObjects().Count());
    297       progress.ProgressValue = (double)jobCount / totalJobCount;
    298 
    299       log.LogMessage(progress.Status);
    300       TryAndRepeat(() => {
    301         if (parentHiveJob != null) {
    302           hiveJob.Job.Id = service.AddChildJob(parentHiveJob.Job.Id, hiveJob.Job, jobData);
    303         } else {
    304           hiveJob.Job.Id = service.AddJob(hiveJob.Job, jobData, groups.ToList());
    305         }
    306       }, -1, "Failed to add job", log);
    307 
    308       foreach (HiveJob child in hiveJob.ChildHiveJobs) {
    309         UploadJobWithChildren(progress, service, child, hiveJob, groups, ref jobCount, totalJobCount, configPluginId, hiveExperimentId, log, isPrivileged);
     280    private void UploadJobWithChildren(IProgress progress, HiveJob hiveJob, HiveJob parentHiveJob, IEnumerable<Guid> groups, int[] jobCount, int totalJobCount, Guid configPluginId, Guid hiveExperimentId, ILog log, bool isPrivileged, CancellationToken cancellationToken) {
     281      jobUploadSemaphore.WaitOne();
     282      try {
     283        cancellationToken.ThrowIfCancellationRequested();
     284        lock (jobCountLocker) {
     285          jobCount[0]++;
     286        }
     287        JobData jobData = null;
     288        List<IPluginDescription> plugins = null;
     289
     290        TryAndRepeat(() => { // workaround for persistence bug (thread-safe access to bitmaps) - remove later
     291          if (hiveJob.ItemJob.ComputeInParallel && (hiveJob.ItemJob.Item is Optimization.Experiment || hiveJob.ItemJob.Item is Optimization.BatchRun)) {
     292            hiveJob.Job.IsParentJob = true;
     293            hiveJob.Job.FinishWhenChildJobsFinished = true;
     294            jobData = hiveJob.GetAsJobData(true, out plugins);
     295          } else {
     296            hiveJob.Job.IsParentJob = false;
     297            hiveJob.Job.FinishWhenChildJobsFinished = false;
     298            jobData = hiveJob.GetAsJobData(false, out plugins);
     299          }
     300        }, 30, "Could not serialize job");
     301        cancellationToken.ThrowIfCancellationRequested();
     302
     303        TryAndRepeat(() => {
     304          if (!cancellationToken.IsCancellationRequested) {
     305            ServiceLocator.Instance.CallHiveService((s) => hiveJob.Job.PluginsNeededIds = PluginUtil.GetPluginDependencies(s, this.onlinePlugins, this.alreadyUploadedPlugins, plugins));
     306          }
     307        }, -1, "Failed to upload plugins");
     308        cancellationToken.ThrowIfCancellationRequested();
     309        hiveJob.Job.PluginsNeededIds.Add(configPluginId);
     310        hiveJob.Job.HiveExperimentId = hiveExperimentId;
     311        hiveJob.Job.IsPrivileged = isPrivileged;
     312
     313        log.LogMessage(string.Format("Uploading job ({0} kb, {1} objects)", jobData.Data.Count() / 1024, hiveJob.ItemJob.GetObjectGraphObjects().Count()));
     314        TryAndRepeat(() => {
     315          if (!cancellationToken.IsCancellationRequested) {
     316            if (parentHiveJob != null) {
     317              hiveJob.Job.Id = ServiceLocator.Instance.CallHiveService((s) => s.AddChildJob(parentHiveJob.Job.Id, hiveJob.Job, jobData));
     318            } else {
     319              hiveJob.Job.Id = ServiceLocator.Instance.CallHiveService((s) => s.AddJob(hiveJob.Job, jobData, groups.ToList()));
     320            }
     321          }
     322        }, 50, "Failed to add job", log);
     323        cancellationToken.ThrowIfCancellationRequested();
     324
     325        lock (jobCountLocker) {
     326          progress.ProgressValue = (double)jobCount[0] / totalJobCount;
     327          progress.Status = string.Format("Uploaded job ({0} of {1})", jobCount[0], totalJobCount);
     328        }
     329
     330        var tasks = new List<Task>();
     331        foreach (HiveJob child in hiveJob.ChildHiveJobs) {
     332          tasks.Add(Task.Factory.StartNew((tuple) => {
     333            var arguments = (Tuple<HiveJob, HiveJob>)tuple;
     334            UploadJobWithChildren(progress, arguments.Item1, arguments.Item2, groups, jobCount, totalJobCount, configPluginId, hiveExperimentId, log, isPrivileged, cancellationToken);
     335          }, new Tuple<HiveJob, HiveJob>(child, hiveJob ))
     336          .ContinueWith((x) => log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted));
     337        }
     338        Task.WaitAll(tasks.ToArray());
     339      }
     340      finally {
     341        jobUploadSemaphore.Release();
    310342      }
    311343    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ServiceClients/HiveItem.cs

    r6373 r6444  
    8686
    8787    public void Store() {
    88       HiveClient.Store(this);
     88      HiveClient.Store(this, new System.Threading.CancellationToken());
    8989      Modified = false;
    9090    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/Util/PluginUtil.cs

    r6426 r6444  
    3030namespace HeuristicLab.Clients.Hive {
    3131  public static class PluginUtil {
     32    private static object locker = new object();
    3233
    3334    /// <summary>
    … …  
    4041    /// <returns></returns>
    4142    public static List<Guid> GetPluginDependencies(IHiveService service, List<Plugin> onlinePlugins, List<Plugin> alreadyUploadedPlugins, IEnumerable<IPluginDescription> neededPlugins) {
    42       var pluginIds = new List<Guid>();
    43       Dictionary<IPluginDescription, byte[]> checksumsNeededPlugins = CalcChecksumsForPlugins(neededPlugins);
     43      lock (locker) {
     44        var pluginIds = new List<Guid>();
     45        Dictionary<IPluginDescription, byte[]> checksumsNeededPlugins = CalcChecksumsForPlugins(neededPlugins);
    4446
    45       foreach (var neededPlugin in checksumsNeededPlugins) {
    46         Plugin foundPlugin = alreadyUploadedPlugins.FirstOrDefault(p => p.Hash.SequenceEqual(neededPlugin.Value));
    47         if (foundPlugin == null) {
    48           foundPlugin = onlinePlugins.FirstOrDefault(p => {
    49             if (p.Hash != null) {
    50               return p.Hash.SequenceEqual(neededPlugin.Value);
     47        foreach (var neededPlugin in checksumsNeededPlugins) {
     48          Plugin foundPlugin = alreadyUploadedPlugins.FirstOrDefault(p => p.Hash.SequenceEqual(neededPlugin.Value));
     49          if (foundPlugin == null) {
     50            foundPlugin = onlinePlugins.FirstOrDefault(p => {
     51              if (p.Hash != null) {
     52                return p.Hash.SequenceEqual(neededPlugin.Value);
     53              } else {
     54                return false;
     55              }
     56            });
     57
     58            if (foundPlugin == null) {
     59              Plugin p = CreatePlugin(neededPlugin.Key, neededPlugin.Value);
     60              List<PluginData> pd = CreatePluginDatas(neededPlugin.Key);
     61              try {
     62                p.Id = service.AddPlugin(p, pd);
     63                alreadyUploadedPlugins.Add(p);
     64                pluginIds.Add(p.Id);
     65              }
     66              catch (FaultException<PluginAlreadyExistsFault> fault) {
     67                onlinePlugins.Add(service.GetPlugin(fault.Detail.Id));
     68              }
    5169            } else {
    52               return false;
    53             }
    54           });
    55 
    56           if (foundPlugin == null) {
    57             Plugin p = CreatePlugin(neededPlugin.Key, neededPlugin.Value);
    58             List<PluginData> pd = CreatePluginDatas(neededPlugin.Key);
    59             try {
    60               p.Id = service.AddPlugin(p, pd);
    61               alreadyUploadedPlugins.Add(p);
    62               pluginIds.Add(p.Id);
    63             }
    64             catch (FaultException<PluginAlreadyExistsFault> fault) {
    65               onlinePlugins.Add(service.GetPlugin(fault.Detail.Id));
     70              pluginIds.Add(foundPlugin.Id);
    6671            }
    6772          } else {
    6873            pluginIds.Add(foundPlugin.Id);
    6974          }
    70         } else {
    71           pluginIds.Add(foundPlugin.Id);
    7275        }
     76        return pluginIds;
    7377      }
    74       return pluginIds;
    7578    }
    7679

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Views/3.4/HiveEngineView.Designer.cs

    r6426 r6444  
    6060      this.resourceIdsTextBox.Location = new System.Drawing.Point(189, 52);
    6161      this.resourceIdsTextBox.Name = "resourceIdsTextBox";
    62       this.resourceIdsTextBox.Size = new System.Drawing.Size(319, 20);
     62      this.resourceIdsTextBox.Size = new System.Drawing.Size(428, 20);
    6363      this.resourceIdsTextBox.TabIndex = 4;
    6464      this.resourceIdsTextBox.Text = "HEAL";
    … …  
    184184      this.isPrivilegedCheckBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
    185185      this.isPrivilegedCheckBox.AutoSize = true;
    186       this.isPrivilegedCheckBox.Location = new System.Drawing.Point(514, 55);
     186      this.isPrivilegedCheckBox.Location = new System.Drawing.Point(623, 54);
    187187      this.isPrivilegedCheckBox.Name = "isPrivilegedCheckBox";
    188188      this.isPrivilegedCheckBox.Size = new System.Drawing.Size(80, 17);
    … …  
    190190      this.isPrivilegedCheckBox.Text = "IsPrivileged";
    191191      this.isPrivilegedCheckBox.UseVisualStyleBackColor = true;
    192       this.isPrivilegedCheckBox.CheckedChanged += new System.EventHandler(this.isPrivilegedCheckBox_CheckedChanged);
     192      this.isPrivilegedCheckBox.Validated += new System.EventHandler(this.isPrivilegedCheckBox_Validated);
    193193      //
    194194      // HiveEngineView

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Views/3.4/HiveEngineView.cs

    r6426 r6444  
    109109    }
    110110
    111     private void isPrivilegedCheckBox_CheckedChanged(object sender, EventArgs e) {
     111    private void isPrivilegedCheckBox_Validated(object sender, EventArgs e) {
    112112      Content.IsPrivileged = isPrivilegedCheckBox.Checked;
    113113    }
    114114    #endregion
    115 
    116 
    117115  }
    118116}

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine/3.4/HiveEngine.cs

    r6435 r6444  
    256256            random.Reset(random.Next());
    257257        }
    258         HiveClient.StartExperiment((e) => { log.LogException(e); }, refreshableHiveExperiment);
     258        HiveClient.StartExperiment((e) => { log.LogException(e); }, refreshableHiveExperiment, cancellationToken);
    259259
    260260        // do polling until experiment is finished and all jobs are downloaded

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/TransactionManager.cs

    r6431 r6444  
    2727  public static class TransactionManager {
    2828    public static void UseTransaction(Action call, bool serializable = false, bool longRunning = false) {
    29       TransactionScope transaction = CreateTransaction(serializable, longRunning);
    30       try {
    31         call();
    32         transaction.Complete();
    33       }
    34       finally {
    35         transaction.Dispose();
     29      int n = 10;
     30      while (n > 0) {
     31        TransactionScope transaction = CreateTransaction(serializable, longRunning);
     32        try {
     33          call();
     34          transaction.Complete();
     35          n = 0;
     36        }
     37        catch (System.Data.SqlClient.SqlException e) {
     38          n--; // probably deadlock situation, let it roll back and repeat the transaction n times
     39          LogFactory.GetLogger(typeof(TransactionManager).Namespace).Log(string.Format("Exception occured, repeating transaction {0} more times. Details: {1}", n, e.ToString()));         
     40          if (n <= 0) throw;
     41        }
     42        finally {
     43          transaction.Dispose();
     44        }
    3645      }
    3746    }
    3847
    3948    public static T UseTransaction<T>(Func<T> call, bool serializable = false, bool longRunning = false) {
    40       TransactionScope transaction = CreateTransaction(serializable, longRunning);
    41       try {
    42         T result = call();
    43         transaction.Complete();
    44         return result;
     49      int n = 10;
     50      while (n > 0) {
     51        TransactionScope transaction = CreateTransaction(serializable, longRunning);
     52        try {
     53          T result = call();
     54          transaction.Complete();
     55          n = 0;
     56          return result;
     57        }
     58        catch (System.Data.SqlClient.SqlException e) {
     59          n--; // probably deadlock situation, let it roll back and repeat the transaction n times
     60          LogFactory.GetLogger(typeof(TransactionManager).Namespace).Log(string.Format("Exception occured, repeating transaction {0} more times. Details: {1}", n, e.ToString()));
     61          if (n <= 0) throw;
     62        }
     63        finally {
     64          transaction.Dispose();
     65        }
    4566      }
    46       finally {
    47         transaction.Dispose();
    48       }
     67      throw new Exception("This code should not be reached");
    4968    }
    50    
     69
    5170    private static TransactionScope CreateTransaction(bool serializable, bool longRunning) {
    5271      var options = new TransactionOptions();
    … …  
    5877      if (longRunning)
    5978        options.Timeout = ApplicationConstants.LongRunningDatabaseCommandTimeout;
    60      
     79
    6180      return new TransactionScope(TransactionScopeOption.Required, options);
    6281    }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HeartbeatManager.cs

    r6431 r6444  
    6262          if (availableJobs.Count() > 0) {
    6363            var job = availableJobs.First();
    64             actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob, job.Id));
    65             AssignJob(slave, job);
     64            if (AssignJob(slave, job))
     65              actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob, job.Id));
    6666          }
    6767        }
    … …  
    7070    }
    7171
    72     private void AssignJob(Slave slave, Job job) {
     72    // returns true if assignment was successful
     73    private bool AssignJob(Slave slave, Job job) {
     74      // load job again and check if it is still available (this is an attempt to reduce the race condition which causes multiple heartbeats to get the same job assigned)
     75      if (dao.GetJob(job.Id).State != JobState.Waiting) return false;
     76
    7377      job = dao.UpdateJobState(job.Id, JobState.Transferring, slave.Id, null, null);
    74       dao.UpdateSlave(slave);
    7578
    7679      // from now on the job has some time to send the next heartbeat (ApplicationConstants.TransferringJobHeartbeatTimeout)
    7780      job.LastHeartbeat = DateTime.Now;
    7881      dao.UpdateJob(job);
     82      return true;
    7983    }
    8084
Note: See TracChangeset for help on using the changeset viewer.