#region License Information /* HeuristicLab * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL) * * This file is part of HeuristicLab. * * HeuristicLab is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * HeuristicLab is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with HeuristicLab. If not, see . */ #endregion using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.Serialization.Formatters.Binary; using System.Threading; using System.Transactions; using HeuristicLab.Hive.Contracts; using HeuristicLab.Hive.Contracts.BusinessObjects; using HeuristicLab.Hive.Contracts.Interfaces; using HeuristicLab.Hive.Server.Core.InternalInterfaces; using HeuristicLab.PluginInfrastructure; using HeuristicLab.Tracing; using HeuristicLab.Hive.Contracts.ResponseObjects; namespace HeuristicLab.Hive.Server.Core { /// /// The ClientCommunicator manages the whole communication with the client /// public class SlaveCommunicator : ISlaveCommunicator, IInternalSlaveCommunicator { private static Dictionary lastHeartbeats = new Dictionary(); private static Dictionary newAssignedJobs = new Dictionary(); private static Dictionary pendingJobs = new Dictionary(); private static ReaderWriterLockSlim heartbeatLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); //private ISessionFactory factory; private ILifecycleManager lifecycleManager; private IInternalJobManager jobManager; private IScheduler scheduler; private static int PENDING_TIMEOUT = 100; /// /// Initialization of the Adapters to the database /// Initialization of Eventhandler for the lifecycle management /// Initialization of lastHearbeats Dictionary /// public SlaveCommunicator() { //factory = ServiceLocator.GetSessionFactory(); lifecycleManager = ServiceLocator.GetLifecycleManager(); jobManager = ServiceLocator.GetJobManager() as IInternalJobManager; scheduler = ServiceLocator.GetScheduler(); lifecycleManager.RegisterHeartbeat(new EventHandler(lifecycleManager_OnServerHeartbeat)); } /// /// Check if online clients send their hearbeats /// if not -> set them offline and check if they where calculating a job /// /// /// void lifecycleManager_OnServerHeartbeat(object sender, EventArgs e) { Logger.Debug("Server Heartbeat ticked"); // [chn] why is transaction management done here using (TransactionScope scope = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = ApplicationConstants.ISOLATION_LEVEL_SCOPE })) { List allClients = new List(DaoLocator.ClientDao.FindAll()); foreach (ClientDto client in allClients) { if (client.State != SlaveState.Offline && client.State != SlaveState.NullState) { heartbeatLock.EnterUpgradeableReadLock(); if (!lastHeartbeats.ContainsKey(client.Id)) { Logger.Info("Client " + client.Id + " wasn't offline but hasn't sent heartbeats - setting offline"); client.State = SlaveState.Offline; DaoLocator.ClientDao.Update(client); Logger.Info("Client " + client.Id + " wasn't offline but hasn't sent heartbeats - Resetting all his jobs"); foreach (JobDto job in DaoLocator.JobDao.FindActiveJobsOfSlave(client)) { //maybe implementa n additional Watchdog? Till then, just set them offline.. DaoLocator.JobDao.SetJobOffline(job); } } else { DateTime lastHbOfClient = lastHeartbeats[client.Id]; TimeSpan dif = DateTime.Now.Subtract(lastHbOfClient); // check if time between last hearbeat and now is greather than HEARTBEAT_MAX_DIF if (dif.TotalSeconds > ApplicationConstants.HEARTBEAT_MAX_DIF) { // if client calculated jobs, the job must be reset Logger.Info("Client timed out and is on RESET"); foreach (JobDto job in DaoLocator.JobDao.FindActiveJobsOfSlave(client)) { DaoLocator.JobDao.SetJobOffline(job); lock (newAssignedJobs) { if (newAssignedJobs.ContainsKey(job.Id)) newAssignedJobs.Remove(job.Id); } } Logger.Debug("setting client offline"); // client must be set offline client.State = SlaveState.Offline; //clientAdapter.Update(client); DaoLocator.ClientDao.Update(client); Logger.Debug("removing it from the heartbeats list"); heartbeatLock.EnterWriteLock(); lastHeartbeats.Remove(client.Id); heartbeatLock.ExitWriteLock(); } } heartbeatLock.ExitUpgradeableReadLock(); } else { //TODO: RLY neccesary? //HiveLogger.Info(this.ToString() + ": Client " + client.Id + " has wrong state: Shouldn't have offline or nullstate, has " + client.State); heartbeatLock.EnterWriteLock(); //HiveLogger.Info(this.ToString() + ": Client " + client.Id + " has wrong state: Resetting all his jobs"); if (lastHeartbeats.ContainsKey(client.Id)) lastHeartbeats.Remove(client.Id); foreach (JobDto job in DaoLocator.JobDao.FindActiveJobsOfSlave(client)) { DaoLocator.JobDao.SetJobOffline(job); } heartbeatLock.ExitWriteLock(); } } CheckForPendingJobs(); // DaoLocator.DestroyContext(); scope.Complete(); } } private void CheckForPendingJobs() { IList pendingJobsInDB = new List(DaoLocator.JobDao.GetJobsByState(JobState.Pending)); foreach (JobDto currJob in pendingJobsInDB) { lock (pendingJobs) { if (pendingJobs.ContainsKey(currJob.Id)) { if (pendingJobs[currJob.Id] <= 0) { currJob.State = JobState.Offline; DaoLocator.JobDao.Update(currJob); } else { pendingJobs[currJob.Id]--; } } } } } #region IClientCommunicator Members /// /// Login process for the client /// A hearbeat entry is created as well (login is the first hearbeat) /// /// /// public Response Login(ClientDto slaveInfo) { Response response = new Response(); heartbeatLock.EnterWriteLock(); if (lastHeartbeats.ContainsKey(slaveInfo.Id)) { lastHeartbeats[slaveInfo.Id] = DateTime.Now; } else { lastHeartbeats.Add(slaveInfo.Id, DateTime.Now); } heartbeatLock.ExitWriteLock(); ClientDto dbClient = DaoLocator.ClientDao.FindById(slaveInfo.Id); //Really set offline? //Reconnect issues with the currently calculating jobs slaveInfo.State = SlaveState.Idle; slaveInfo.CalendarSyncStatus = dbClient != null ? dbClient.CalendarSyncStatus : CalendarState.NotAllowedToFetch; if (dbClient == null) DaoLocator.ClientDao.Insert(slaveInfo); else DaoLocator.ClientDao.Update(slaveInfo); return response; } public ResponseCalendar GetCalendar(Guid clientId) { ResponseCalendar response = new ResponseCalendar(); ClientDto client = DaoLocator.ClientDao.FindById(clientId); if (client == null) { //response.Success = false; response.StatusMessage = ResponseStatus.GetCalendar_ResourceNotFound; return response; } response.ForceFetch = (client.CalendarSyncStatus == CalendarState.ForceFetch); IEnumerable appointments = DaoLocator.UptimeCalendarDao.GetCalendarForClient(client); if (appointments.Count() == 0) { response.StatusMessage = ResponseStatus.GetCalendar_NoCalendarFound; //response.Success = false; } else { //response.Success = true; response.Appointments = appointments; } client.CalendarSyncStatus = CalendarState.Fetched; DaoLocator.ClientDao.Update(client); return response; } public Response SetCalendarStatus(Guid clientId, CalendarState state) { Response response = new Response(); ClientDto client = DaoLocator.ClientDao.FindById(clientId); if (client == null) { //response.Success = false; response.StatusMessage = ResponseStatus.GetCalendar_ResourceNotFound; return response; } client.CalendarSyncStatus = state; DaoLocator.ClientDao.Update(client); return response; } /// /// The client has to send regulary heartbeats /// this hearbeats will be stored in the heartbeats dictionary /// check if there is work for the client and send the client a response if he should pull a job /// /// /// public ResponseHeartBeat ProcessHeartBeat(HeartBeatData hbData) { Logger.Debug("BEGIN Processing Heartbeat for Client " + hbData.SlaveId); ResponseHeartBeat response = new ResponseHeartBeat(); response.ActionRequest = new List(); Logger.Debug("BEGIN Started Client Fetching"); ClientDto client = DaoLocator.ClientDao.FindById(hbData.SlaveId); Logger.Debug("END Finished Client Fetching"); // check if the client is logged in if (client.State == SlaveState.Offline || client.State == SlaveState.NullState) { // response.Success = false; response.StatusMessage = ResponseStatus.ProcessHeartBeat_UserNotLoggedIn; response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.NoMessage)); Logger.Error("ProcessHeartBeat: Client state null or offline: " + client); return response; } client.NrOfFreeCores = hbData.FreeCores; client.FreeMemory = hbData.FreeMemory; // save timestamp of this heartbeat Logger.Debug("BEGIN Locking for Heartbeats"); heartbeatLock.EnterWriteLock(); Logger.Debug("END Locked for Heartbeats"); if (lastHeartbeats.ContainsKey(hbData.SlaveId)) { lastHeartbeats[hbData.SlaveId] = DateTime.Now; } else { lastHeartbeats.Add(hbData.SlaveId, DateTime.Now); } heartbeatLock.ExitWriteLock(); Logger.Debug("BEGIN Processing Heartbeat Jobs"); ProcessJobProcess(hbData, response); Logger.Debug("END Processed Heartbeat Jobs"); //check if new Cal must be loaded if (client.CalendarSyncStatus == CalendarState.Fetch || client.CalendarSyncStatus == CalendarState.ForceFetch) { response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.FetchOrForceFetchCalendar)); //client.CalendarSyncStatus = CalendarState.Fetching; Logger.Info("fetch or forcefetch sent"); } // check if client has a free core for a new job // if true, ask scheduler for a new job for this client Logger.Debug(" BEGIN Looking for Client Jobs"); if (hbData.FreeCores > 0 && scheduler.ExistsJobForSlave(hbData)) { response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.FetchJob)); } else { response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.NoMessage)); } Logger.Debug(" END Looked for Client Jobs"); DaoLocator.ClientDao.Update(client); //tx.Commit(); Logger.Debug(" END Processed Heartbeat for Client " + hbData.SlaveId); return response; } /// /// Process the Job progress sent by a client /// [chn] this method needs to be refactored, because its a performance hog /// /// what it does: /// (1) find out if the jobs that should be calculated by this client (from db) and compare if they are consistent with what the joblist the client sent /// (2) find out if every job from the joblist really should be calculated by this client /// (3) checks if a job should be aborted and issues Message /// (4) update job-progress and write to db /// (5) if snapshot is requested, issue Message /// /// (6) for each job from DB, check if there is a job from client (again). /// (7) if job matches, it is removed from newAssigneJobs /// (8) if job !matches, job's TTL is reduced by 1, /// (9) if TTL==0, job is set to Abort (save to DB), and Message to Abort job is issued to client /// /// /// /// quirks: /// (1) the response-object is modified during the foreach-loop (only last element counts) /// (2) state Abort results in Finished. This should be: AbortRequested, Aborted. /// /// /// /// /// private void ProcessJobProcess(HeartBeatData hbData, ResponseHeartBeat response) { Logger.Debug("Started for Client " + hbData.SlaveId); List jobsOfClient = new List(DaoLocator.JobDao.FindActiveJobsOfSlave(DaoLocator.ClientDao.FindById(hbData.SlaveId))); if (hbData.JobProgress != null && hbData.JobProgress.Count > 0) { if (jobsOfClient == null || jobsOfClient.Count == 0) { //response.Success = false; //response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_IS_NOT_BEEING_CALCULATED; foreach (Guid jobId in hbData.JobProgress.Keys) { response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, jobId)); } Logger.Error("There is no job calculated by this user " + hbData.SlaveId + ", advise him to abort all"); return; } foreach (KeyValuePair jobProgress in hbData.JobProgress) { JobDto curJob = DaoLocator.JobDao.FindById(jobProgress.Key); curJob.Client = DaoLocator.ClientDao.GetClientForJob(curJob.Id); if (curJob.Client == null || curJob.Client.Id != hbData.SlaveId) { //response.Success = false; //response.StatusMessage = ApplicationConstants.RESPONSE_COMMUNICATOR_JOB_IS_NOT_BEEING_CALCULATED; response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, curJob.Id)); Logger.Error("There is no job calculated by this user " + hbData.SlaveId + " Job: " + curJob); } else if (curJob.State == JobState.Aborted) { // a request to abort the job has been set response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, curJob.Id)); curJob.State = JobState.Finished; } else { // save job progress curJob.Percentage = jobProgress.Value; if (curJob.State == JobState.SnapshotRequested) { // a request for a snapshot has been set response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.RequestSnapshot, curJob.Id)); curJob.State = JobState.SnapshotSent; } } DaoLocator.JobDao.Update(curJob); } } foreach (JobDto currJob in jobsOfClient) { bool found = false; if (hbData.JobProgress != null) { foreach (Guid jobId in hbData.JobProgress.Keys) { if (jobId == currJob.Id) { found = true; break; } } } if (!found) { lock (newAssignedJobs) { if (newAssignedJobs.ContainsKey(currJob.Id)) { newAssignedJobs[currJob.Id]--; Logger.Error("Job TTL Reduced by one for job: " + currJob + "and is now: " + newAssignedJobs[currJob.Id] + ". User that sucks: " + currJob.Client); if (newAssignedJobs[currJob.Id] <= 0) { Logger.Error("Job TTL reached Zero, Job gets removed: " + currJob + " and set back to offline. User that sucks: " + currJob.Client); currJob.State = JobState.Offline; DaoLocator.JobDao.Update(currJob); response.ActionRequest.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, currJob.Id)); newAssignedJobs.Remove(currJob.Id); } } else { Logger.Error("Job ID wasn't with the heartbeats: " + currJob); currJob.State = JobState.Offline; DaoLocator.JobDao.Update(currJob); } } // lock } else { lock (newAssignedJobs) { if (newAssignedJobs.ContainsKey(currJob.Id)) { Logger.Info("Job is sending a heart beat, removing it from the newAssignedJobList: " + currJob); newAssignedJobs.Remove(currJob.Id); } } } } } /// /// if the client was told to pull a job he calls this method /// the server selects a job and sends it to the client /// /// /// public ResponseObject GetJob(Guid clientId) { ResponseObject response = new ResponseObject(); JobDto job2Calculate = scheduler.GetNextJobForSlave(clientId); if (job2Calculate != null) { response.Obj = job2Calculate; response.Obj.PluginsNeeded = DaoLocator.PluginInfoDao.GetPluginDependenciesForJob(response.Obj); Logger.Info("Job pulled: " + job2Calculate + " for user " + clientId); lock (newAssignedJobs) { if (!newAssignedJobs.ContainsKey(job2Calculate.Id)) newAssignedJobs.Add(job2Calculate.Id, ApplicationConstants.JOB_TIME_TO_LIVE); } } else { //response.Success = false; response.Obj = null; response.StatusMessage = ResponseStatus.GetJob_NoJobsAvailable; Logger.Info("No more Jobs left for " + clientId); } return response; } public ResponseResultReceived ProcessJobResult(Stream stream, bool finished) { Logger.Info("BEGIN Job received for Storage - main method:"); //Stream jobResultStream = null; //Stream jobStream = null; //try { BinaryFormatter formatter = new BinaryFormatter(); JobResult result = (JobResult)formatter.Deserialize(stream); //important - repeatable read isolation level is required here, //otherwise race conditions could occur when writing the stream into the DB //just removed TransactionIsolationLevel.RepeatableRead //tx = session.BeginTransaction(); ResponseResultReceived response = ProcessJobResult(result.ClientId, result.JobId, new byte[] { }, result.Percentage, result.Exception, finished); if (response.StatusMessage == ResponseStatus.Ok) { Logger.Debug("Trying to aquire WCF Job Stream"); //jobStream = DaoLocator.JobDao.GetSerializedJobStream(result.JobId); //Logger.Debug("Job Stream Aquired"); byte[] buffer = new byte[3024]; List serializedJob = new List(); int read = 0; int i = 0; while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) { for (int j = 0; j < read; j++) { serializedJob.Add(buffer[j]); } if (i % 100 == 0) Logger.Debug("Writing to stream: " + i); //jobStream.Write(buffer, 0, read); i++; } Logger.Debug("Done Writing, closing the stream!"); //jobStream.Close(); DaoLocator.JobDao.SetBinaryJobFile(result.JobId, serializedJob.ToArray()); } Logger.Info("END Job received for Storage:"); stream.Dispose(); return response; } private ResponseResultReceived ProcessJobResult(Guid clientId, Guid jobId, byte[] result, double? percentage, string exception, bool finished) { Logger.Info("BEGIN Job received for Storage - SUB method: " + jobId); ResponseResultReceived response = new ResponseResultReceived(); ClientDto client = DaoLocator.ClientDao.FindById(clientId); SerializedJob job = new SerializedJob(); if (job != null) { job.JobInfo = DaoLocator.JobDao.FindById(jobId); if (job.JobInfo != null) { job.JobInfo.Client = job.JobInfo.Client = DaoLocator.ClientDao.GetClientForJob(jobId); } } if (job != null && job.JobInfo == null) { //response.Success = false; response.StatusMessage = ResponseStatus.ProcessJobResult_JobDoesNotExist; response.JobId = jobId; Logger.Error("No job with Id " + jobId); //tx.Rollback(); return response; } if (job.JobInfo.State == JobState.Aborted) { //response.Success = false; response.StatusMessage = ResponseStatus.ProcessJobResult_JobAborted; Logger.Error("Job was aborted! " + job.JobInfo); //tx.Rollback(); return response; } if (job.JobInfo.Client == null) { //response.Success = false; response.StatusMessage = ResponseStatus.ProcessJobResult_JobIsNotBeeingCalculated; response.JobId = jobId; Logger.Error("Job is not being calculated (client = null)! " + job.JobInfo); //tx.Rollback(); return response; } if (job.JobInfo.Client.Id != clientId) { //response.Success = false; response.StatusMessage = ResponseStatus.ProcessJobResult_WrongClientForJob; response.JobId = jobId; Logger.Error("Wrong Client for this Job! " + job.JobInfo + ", Sending Client is: " + clientId); //tx.Rollback(); return response; } if (job.JobInfo.State == JobState.Finished) { response.StatusMessage = ResponseStatus.Ok; response.JobId = jobId; Logger.Error("Job already finished! " + job.JobInfo + ", Sending Client is: " + clientId); //tx.Rollback(); return response; } //Todo: RequestsnapshotSent => calculating? if (job.JobInfo.State == JobState.SnapshotSent) { job.JobInfo.State = JobState.Calculating; } if (job.JobInfo.State != JobState.Calculating && job.JobInfo.State != JobState.Pending) { //response.Success = false; response.StatusMessage = ResponseStatus.ProcessJobResult_InvalidJobState; response.JobId = jobId; Logger.Error("Wrong Job State, job is: " + job.JobInfo); //tx.Rollback(); return response; } job.JobInfo.Percentage = percentage; if (!string.IsNullOrEmpty(exception)) { job.JobInfo.State = JobState.Failed; job.JobInfo.Exception = exception; job.JobInfo.DateFinished = DateTime.Now; } else if (finished) { job.JobInfo.State = JobState.Finished; job.JobInfo.DateFinished = DateTime.Now; } job.SerializedJobData = result; DaoLocator.JobDao.Update(job.JobInfo); response.StatusMessage = ResponseStatus.Ok; response.JobId = jobId; response.Finished = finished; Logger.Info("END Job received for Storage - SUB method: " + jobId); return response; } /// /// the client can send job results during calculating /// and will send a final job result when he finished calculating /// these job results will be stored in the database /// /// /// /// /// /// /// public ResponseResultReceived StoreFinishedJobResult(Guid clientId, Guid jobId, byte[] result, double percentage, string exception) { return ProcessJobResult(clientId, jobId, result, percentage, exception, true); } public ResponseResultReceived ProcessSnapshot(Guid clientId, Guid jobId, byte[] result, double percentage, string exception) { return ProcessJobResult(clientId, jobId, result, percentage, exception, false); } /// /// when a client logs out the state will be set /// and the entry in the last hearbeats dictionary will be removed /// /// /// public Response Logout(Guid clientId) { Logger.Info("Client logged out " + clientId); Response response = new Response(); heartbeatLock.EnterWriteLock(); if (lastHeartbeats.ContainsKey(clientId)) lastHeartbeats.Remove(clientId); heartbeatLock.ExitWriteLock(); ClientDto client = DaoLocator.ClientDao.FindById(clientId); if (client == null) { //response.Success = false; response.StatusMessage = ResponseStatus.Logout_SlaveNotRegistered; return response; } if (client.State == SlaveState.Calculating) { // check wich job the client was calculating and reset it IEnumerable jobsOfClient = DaoLocator.JobDao.FindActiveJobsOfSlave(client); foreach (JobDto job in jobsOfClient) { if (job.State != JobState.Finished) DaoLocator.JobDao.SetJobOffline(job); } } client.State = SlaveState.Offline; DaoLocator.ClientDao.Update(client); return response; } /// /// If a client goes offline and restores a job he was calculating /// he can ask the client if he still needs the job result /// /// /// public Response IsJobStillNeeded(Guid jobId) { Response response = new Response(); JobDto job = DaoLocator.JobDao.FindById(jobId); if (job == null) { //response.Success = false; response.StatusMessage = ResponseStatus.IsJobStillNeeded_JobDoesNotExist; Logger.Error("Job doesn't exist (anymore)! " + jobId); return response; } if (job.State == JobState.Finished) { //response.Success = true; response.StatusMessage = ResponseStatus.IsJobStillNeeded_JobAlreadyFinished; Logger.Error("already finished! " + job); return response; } job.State = JobState.Pending; lock (pendingJobs) { pendingJobs.Add(job.Id, PENDING_TIMEOUT); } DaoLocator.JobDao.Update(job); return response; } public ResponseList GetPlugins(List pluginList) { ResponseList response = new ResponseList(); response.List = new List(); foreach (HivePluginInfoDto pluginInfo in pluginList) { if (pluginInfo.Update) { //check if there is a newer version IPluginDescription ipd = ApplicationManager.Manager.Plugins.Where(pd => pd.Name == pluginInfo.Name && pd.Version.Major == pluginInfo.Version.Major && pd.Version.Minor == pluginInfo.Version.Minor && pd.Version.Revision > pluginInfo.Version.Revision).SingleOrDefault(); if (ipd != null) { response.List.Add(ConvertPluginDescriptorToDto(ipd)); } } else { IPluginDescription ipd = ApplicationManager.Manager.Plugins.Where(pd => pd.Name == pluginInfo.Name && pd.Version.Major == pluginInfo.Version.Major && pd.Version.Minor == pluginInfo.Version.Minor && pd.Version.Revision >= pluginInfo.Version.Revision).SingleOrDefault(); if (ipd != null) { response.List.Add(ConvertPluginDescriptorToDto(ipd)); } else { //response.Success = false; response.StatusMessage = ResponseStatus.GetPlugins_PluginsNotAvailable; return response; } } } return response; } private CachedHivePluginInfoDto ConvertPluginDescriptorToDto(IPluginDescription currPlugin) { CachedHivePluginInfoDto currCachedPlugin = new CachedHivePluginInfoDto { Name = currPlugin.Name, Version = currPlugin.Version }; foreach (string fileName in from file in currPlugin.Files select file.Name) { currCachedPlugin.PluginFiles.Add(new HivePluginFile(File.ReadAllBytes(fileName), fileName)); } return currCachedPlugin; } #endregion } }