Changeset 2608

Timestamp:

01/06/10 14:30:05 (14 years ago)

Author:

kgrading

Message:

changed a ton of logging, changed minor job handling and changed the DB IP to localhost (#828)

Location:

trunk/sources/HeuristicLab.Hive.Server.Core/3.2

Files:

• ClassDiagram1.cd (added)
• ClientCommunicator.cs (modified) (32 diffs)
• HeuristicLab.Hive.Server.Core-3.2.csproj (modified) (2 diffs)
• JobManager.cs (modified) (3 diffs)
• Properties/Settings.Designer.cs (modified) (2 diffs)
• Properties/Settings.settings (modified) (1 diff)
• app.config (modified) (1 diff)

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/ClientCommunicator.cs

@@ -38 +38 @@
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
+using HeuristicLab.Tracing;
 
 namespace HeuristicLab.Hive.Server.Core {
@@ -89 +90 @@
       ITransaction tx = null;
 
+      HiveLogger.Info(this.ToString() + ": Server Heartbeat ticked");
+
       try {
         IClientAdapter clientAdapter =
@@ -104 +107 @@
 
             if (!lastHeartbeats.ContainsKey(client.Id)) {
+              HiveLogger.Info(this.ToString() + ": Client " + client.Id + " wasn't offline but hasn't sent heartbeats - setting offline");
               client.State = State.offline;
               clientAdapter.Update(client);
-              foreach (Job job in jobAdapter.GetActiveJobsOf(client)) {
+              HiveLogger.Info(this.ToString() + ": Client " + client.Id + " wasn't offline but hasn't sent heartbeats - Resetting all his jobs");
+              foreach (Job job in jobAdapter.GetActiveJobsOf(client)) {                
                 jobManager.ResetJobsDependingOnResults(job);
               }
@@ -116 +121 @@
               if (dif.TotalSeconds > ApplicationConstants.HEARTBEAT_MAX_DIF) {
                 // if client calculated jobs, the job must be reset
+                HiveLogger.Info(this.ToString() + ": Client timed out and is on RESET");
                 foreach (Job job in jobAdapter.GetActiveJobsOf(client)) {
                   jobManager.ResetJobsDependingOnResults(job);
@@ -136 +142 @@
             heartbeatLock.ExitUpgradeableReadLock();
           } else {
+            //TODO: RLY neccesary?
+            //HiveLogger.Info(this.ToString() + ": Client " + client.Id + " has wrong state: Shouldn't have offline or nullstate, has " + client.State);
             heartbeatLock.EnterWriteLock();
+            //HiveLogger.Info(this.ToString() + ": Client " + client.Id + " has wrong state: Resetting all his jobs");
             if (lastHeartbeats.ContainsKey(client.Id))
               lastHeartbeats.Remove(client.Id);
@@ -232 +241 @@
     /// <returns></returns>
     public ResponseHB ProcessHeartBeat(HeartBeatData hbData) {
+      
       ISession session = factory.GetSessionForCurrentThread();
       ITransaction tx = null;
 
+      HiveLogger.Info(this.ToString() + ": BEGIN Processing Heartbeat for Client " + hbData.ClientId);
+
       try {
+        HiveLogger.Info(this.ToString() + ": BEGIN Fetching Adapters");
         IClientAdapter clientAdapter =
           session.GetDataAdapter<ClientInfo, IClientAdapter>();
 
-        IJobAdapter jobAdapter =
+        IJobAdapter jobAdapter =        
           session.GetDataAdapter<Job, IJobAdapter>();
-
+        HiveLogger.Info(this.ToString() + ": END Fetched Adapters");
+        HiveLogger.Info(this.ToString() + ": BEGIN Starting Transaction");
         tx = session.BeginTransaction();
+        HiveLogger.Info(this.ToString() + ": END Started Transaction");
 
         ResponseHB response = new ResponseHB();
         response.ActionRequest = new List<MessageContainer>();
 
+        HiveLogger.Info(this.ToString() + ": BEGIN Started Client Fetching");
         ClientInfo client = clientAdapter.GetById(hbData.ClientId);
-
+        HiveLogger.Info(this.ToString() + ": END Finished Client Fetching");
         // check if the client is logged in
         if (client.State == State.offline || client.State == State.nullState) {
           response.Success = false;
-          response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_USER_NOT_LOGGED_IN;
+          response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_USER_NOT_LOGGED_IN;          
           response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.NoMessage));
+
+          HiveLogger.Error(this.ToString() + " ProcessHeartBeat: Client state null or offline: " + client);
+
           return response;
         }
@@ -261 +280 @@
 
         // save timestamp of this heartbeat
+        HiveLogger.Info(this.ToString() + ": BEGIN Locking for Heartbeats");
         heartbeatLock.EnterWriteLock();
+        HiveLogger.Info(this.ToString() + ": END Locked for Heartbeats");
         if (lastHeartbeats.ContainsKey(hbData.ClientId)) {
           lastHeartbeats[hbData.ClientId] = DateTime.Now;
@@ -271 +292 @@
         // check if client has a free core for a new job
         // if true, ask scheduler for a new job for this client
-        if (hbData.FreeCores > 0 && scheduler.ExistsJobForClient(hbData))
+        HiveLogger.Info(this.ToString() + ": BEGIN Looking for Client Jobs");
+        if (hbData.FreeCores > 0 && scheduler.ExistsJobForClient(hbData)) {
           response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.FetchJob));
-        else
+        } else {
           response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.NoMessage));
-
+        }
+        HiveLogger.Info(this.ToString() + ": END Looked for Client Jobs");
         response.Success = true;
         response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_HEARTBEAT_RECEIVED;
 
+        HiveLogger.Info(this.ToString() + ": BEGIN Processing Heartbeat Jobs");
         processJobProcess(hbData, jobAdapter, clientAdapter, response);
+        HiveLogger.Info(this.ToString() + ": END Processed Heartbeat Jobs");
+        
         clientAdapter.Update(client);
 
         tx.Commit();
+        HiveLogger.Info(this.ToString() + ": END Processed Heartbeat for Client " + hbData.ClientId);
         return response;
       }
@@ -304 +331 @@
     /// <param name="response"></param>
     private void processJobProcess(HeartBeatData hbData, IJobAdapter jobAdapter, IClientAdapter clientAdapter, ResponseHB response) {
+      HiveLogger.Info(this.ToString() + " processJobProcess: Started for Client " + hbData.ClientId);      
+      
       if (hbData.JobProgress != null && hbData.JobProgress.Count > 0) {
         List<Job> jobsOfClient = new List<Job>(jobAdapter.GetActiveJobsOf(clientAdapter.GetById(hbData.ClientId)));
@@ -309 +338 @@
           response.Success = false;
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_IS_NOT_BEEING_CALCULATED;
+          HiveLogger.Error(this.ToString() + " processJobProcess: There is no job calculated by this user " + hbData.ClientId);      
           return;
         }
@@ -317 +347 @@
             response.Success = false;
             response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_IS_NOT_BEEING_CALCULATED;
+            HiveLogger.Error(this.ToString() + " processJobProcess: There is no job calculated by this user " + hbData.ClientId + " Job: " + curJob);      
           } else if (curJob.State == State.abort) {
             // a request to abort the job has been set
@@ -345 +376 @@
               if (newAssignedJobs.ContainsKey(currJob.Id)) {
                 newAssignedJobs[currJob.Id]--;
-
+                HiveLogger.Error(this.ToString() + " processJobProcess: Job TTL Reduced by one for job: " + currJob + "and is now: " + newAssignedJobs[currJob.Id] + ". User that sucks: " + currJob.Client);                      
                 if (newAssignedJobs[currJob.Id] <= 0) {
+                  HiveLogger.Error(this.ToString() + " processJobProcess: Job TTL reached Zero, Job gets removed: " + currJob + " and set back to offline. User that sucks: " + currJob.Client);                  
+
                   currJob.State = State.offline;
                   jobAdapter.Update(currJob);
+
+                  response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, currJob.Id));
+
                   newAssignedJobs.Remove(currJob.Id);
                 }
               } else {
+                HiveLogger.Error(this.ToString() + " processJobProcess: Job ID wasn't with the heartbeats:  " + currJob);                      
                 currJob.State = State.offline;
                 jobAdapter.Update(currJob);
@@ -358 +395 @@
           } else {
             lock (newAssignedJobs) {
-              if (newAssignedJobs.ContainsKey(currJob.Id))
+
+              if (newAssignedJobs.ContainsKey(currJob.Id)) {
+                HiveLogger.Info(this.ToString() + " processJobProcess: Job is sending a heart beat, removing it from the newAssignedJobList: " + currJob);                      
                 newAssignedJobs.Remove(currJob.Id);
+              }
             }
           }
@@ -391 +431 @@
           response.Job = computableJob;
           response.Success = true;
+          HiveLogger.Info(this.ToString() + " SendSerializedJob: Job pulled: " + computableJob.JobInfo + " for user " + clientId);                      
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_PULLED;
           lock (newAssignedJobs) {
@@ -397 +438 @@
           }
         } else {
+          HiveLogger.Info(this.ToString() + " SendSerializedJob: No more Jobs left for " + clientId);                      
           response.Success = false;
           response.Job = null;
@@ -439 +481 @@
           response.Job = job2Calculate;
           response.Success = true;
+          HiveLogger.Info(this.ToString() + " SendSerializedJob: Job pulled: " + job2Calculate + " for user " + clientId);                      
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_PULLED;
           lock (newAssignedJobs) {
@@ -448 +491 @@
           response.Job = null;
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_NO_JOBS_LEFT;
+          HiveLogger.Info(this.ToString() + " SendSerializedJob: No more Jobs left for " + clientId);                      
         }
 
@@ -468 +512 @@
       Stream stream,
       bool finished) {
+
+      HiveLogger.Info(this.ToString() + " ProcessJobResult: BEGIN Job received for Storage - main method:");
+      
       ISession session = factory.GetSessionForCurrentThread();
       ITransaction tx = null;
@@ -482 +529 @@
         //important - repeatable read isolation level is required here, 
         //otherwise race conditions could occur when writing the stream into the DB
-        tx = session.BeginTransaction(
-          TransactionIsolationLevel.RepeatableRead);
+        //just removed TransactionIsolationLevel.RepeatableRead
+        tx = session.BeginTransaction();
 
         ResponseResultReceived response =
@@ -520 +567 @@
           tx.Commit();
         }
-
+        HiveLogger.Info(this.ToString() + " ProcessJobResult: END Job received for Storage:");
         return response;
       }
       catch (Exception ex) {
+        HiveLogger.Error(this.ToString() + " ProcessJobResult: Exception raised: " + ex);
         if (tx != null)
           tx.Rollback();
@@ -546 +594 @@
       Exception exception,
       bool finished) {
+      
+      HiveLogger.Info(this.ToString() + " ProcessJobResult: BEGIN Job received for Storage - SUB method: " + jobId);
+
       ISession session = factory.GetSessionForCurrentThread();
+            
       ITransaction tx = null;
 
@@ -556 +608 @@
         IJobResultsAdapter jobResultAdapter =
           session.GetDataAdapter<JobResult, IJobResultsAdapter>();
-
+        
+        //should fetch the existing transaction        
         tx = session.BeginTransaction();
 
@@ -575 +628 @@
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_NO_JOB_WITH_THIS_ID;
           response.JobId = jobId;
+          
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: No job with Id " + jobId);                      
+          
           tx.Rollback();
           return response;
@@ -581 +637 @@
           response.Success = false;
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_WAS_ABORTED;
+          
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: Job was aborted! " + job.JobInfo);                      
+          
           tx.Rollback();
           return response;
@@ -588 +647 @@
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_IS_NOT_BEEING_CALCULATED;
           response.JobId = jobId;
+
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: Job is not being calculated (client = null)! " + job.JobInfo);                      
+
           tx.Rollback();
           return response;
@@ -595 +657 @@
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_WRONG_CLIENT_FOR_JOB;
           response.JobId = jobId;
+
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: Wrong Client for this Job! " + job.JobInfo + ", Sending Client is: " + clientId);                      
+
           tx.Rollback();
           return response;
@@ -602 +667 @@
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOBRESULT_RECEIVED;
           response.JobId = jobId;
+
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: Job already finished! " + job.JobInfo + ", Sending Client is: " + clientId);                      
+
           tx.Rollback();
           return response;
@@ -613 +681 @@
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_WRONG_JOB_STATE;
           response.JobId = jobId;
+
+          HiveLogger.Error(this.ToString() + " ProcessJobResult: Wrong Job State, job is: " + job.JobInfo);                      
+
           tx.Rollback();
           return response;
@@ -649 +720 @@
         response.finished = finished;
         response.JobResultId = jobResult.Id;
-
-        tx.Commit();
+        
+        //Removed for Testing
+        //tx.Commit();
+        HiveLogger.Info(this.ToString() + " ProcessJobResult: END Job received for Storage - SUB method: " + jobId);        
         return response;
       }
@@ -697 +770 @@
     /// <returns></returns>                       
     public Response Logout(Guid clientId) {
+
+      HiveLogger.Info("Client logged out " + clientId);
+      
       ISession session = factory.GetSessionForCurrentThread();
       ITransaction tx = null;
@@ -770 +846 @@
           response.Success = false;
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_DOESNT_EXIST;
+          HiveLogger.Error(this.ToString() + " IsJobStillNeeded: Job doesn't exist (anymore)! " + jobId);
           return response;
         }
@@ -775 +852 @@
           response.Success = true;
           response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_ALLREADY_FINISHED;
+          HiveLogger.Error(this.ToString() + " IsJobStillNeeded: already finished! " + job);
           return response;
         }

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/HeuristicLab.Hive.Server.Core-3.2.csproj

@@ -117 +117 @@
   <ItemGroup>
     <None Include="app.config" />
+    <None Include="ClassDiagram1.cd" />
     <None Include="HeuristicLab.snk" />
     <EmbeddedResource Include="Authorization\HivePermissionSet.xsd">
@@ -156 +157 @@
       <Name>HeuristicLab.Security.Contracts-3.2</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Tracing\3.2\HeuristicLab.Tracing-3.2.csproj">
+      <Project>{EE2034D9-6E27-48A1-B855-42D45F69A4FC}</Project>
+      <Name>HeuristicLab.Tracing-3.2</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/JobManager.cs

@@ -32 +32 @@
 using System.Data;
 using System.IO;
+using HeuristicLab.Tracing;
 
 namespace HeuristicLab.Hive.Server.Core {
@@ -65 +66 @@
 
     public void ResetJobsDependingOnResults(Job job) {
+
+      HiveLogger.Info(this.ToString() + ": Setting job " + job.Id + " offline");
+
       ISession session = factory.GetSessionForCurrentThread();
       ITransaction tx = null;
@@ -122 +126 @@
 
     void checkForDeadJobs() {
+      HiveLogger.Info(this.ToString() + " Searching for dead Jobs");
        ISession session = factory.GetSessionForCurrentThread();
 

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/Properties/Settings.Designer.cs

@@ -2 +2 @@
 // <auto-generated>
 //     This code was generated by a tool.
-//     Runtime Version:2.0.50727.3053
+//     Runtime Version:2.0.50727.4927
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
@@ -26 +26 @@
         [global::System.Configuration.ApplicationScopedSettingAttribute()]
         [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
-        [global::System.Configuration.DefaultSettingValueAttribute("Data Source=10.20.71.10;Initial Catalog=HeuristicLab.Hive;Persist Security Info=T" +
-            "rue;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true")]
+        [global::System.Configuration.DefaultSettingValueAttribute("Data Source=127.0.0.1;Initial Catalog=HeuristicLab.Hive;Persist Security Info=Tru" +
+            "e;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true")]
         public string HiveServerConnectionString {
             get {

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/Properties/Settings.settings

@@ -4 +4 @@
   <Settings>
     <Setting Name="HiveServerConnectionString" Type="System.String" Scope="Application">
-      <Value Profile="(Default)">Data Source=10.20.71.10;Initial Catalog=HeuristicLab.Hive;Persist Security Info=True;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true</Value>
+      <Value Profile="(Default)">Data Source=127.0.0.1;Initial Catalog=HeuristicLab.Hive;Persist Security Info=True;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true</Value>
     </Setting>
   </Settings>

trunk/sources/HeuristicLab.Hive.Server.Core/3.2/app.config

@@ -9 +9 @@
     <HeuristicLab.Hive.Server.Core.Properties.Settings>
       <setting name="HiveServerConnectionString" serializeAs="String">
-        <value>Data Source=10.20.71.10;Initial Catalog=HeuristicLab.Hive;Persist Security Info=True;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true</value>
+        <value>Data Source=127.0.0.1;Initial Catalog=HeuristicLab.Hive;Persist Security Info=True;User ID=hive;Password=hive;Pooling=true;MultipleActiveResultSets=true</value>
       </setting>
     </HeuristicLab.Hive.Server.Core.Properties.Settings>