Changeset 14253 for branches/thasling/DistributedGA

Timestamp:

08/12/16 13:30:41 (8 years ago)

Author:

gkronber

Message:

bugfixing (memory leaks)

Location:

branches/thasling/DistributedGA

Files:

• DistributedGA.Core/DistributedGA.Core.csproj (modified) (1 diff)
• DistributedGA.Core/Implementation/PeerNetworkMessageHandler.cs (modified) (9 diffs)
• DistributedGA.Core/Implementation/WcfMessageSender.cs (modified) (8 diffs)
• DistributedGA.Core/Implementation/WcfPeerListManager.cs (modified) (3 diffs)
• DistributedGA.Core/Interface/IMessageHandler.cs (modified) (2 diffs)
• DistributedGA.Core/Interface/IMessageSender.cs (modified) (2 diffs)
• DistributedGA.Core/Util/ByteArrayWrapper.cs (added)
• DistributedGA.Hive/P2PMigrationAnalyzer.cs (modified) (7 diffs)
• DistributedGA.sln (modified) (2 diffs)

branches/thasling/DistributedGA/DistributedGA.Core/DistributedGA.Core.csproj

@@ -64 +64 @@
     <Compile Include="Interface\IPeerListManager.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Util\ByteArrayWrapper.cs" />
     <Compile Include="Util\SizedConcurrentQueue.cs" />
   </ItemGroup>

branches/thasling/DistributedGA/DistributedGA.Core/Implementation/PeerNetworkMessageHandler.cs

@@ -22 +22 @@
     //two queues are used to gather and and provide population more efficiently
     private object activeQueueLocker = new Object();
-    private SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>> writeQueue;
-    private SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>> readQueue;
+    private SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>> writeQueue;
+    private SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>> readQueue;
 
     //uses IMessageService for recieving population from one peer at once
     private IMessageService host = null;
-
-    private double communicationRate;
 
     public event EventHandler<Exception> ExceptionOccurend;
@@ -40 +38 @@
         }; // TODO: get own peerinfo
 
-        this.communicationRate = communicationRate;
-
-        writeQueue = new SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>>();
-        readQueue = new SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>>();
+        writeQueue = new SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
+        readQueue = new SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
         writeQueue.Limit = messageCacheCapacity;
         readQueue.Limit = writeQueue.Limit;
@@ -57 +53 @@
         sender.Init(ownInstance, messageCacheCapacity);
 
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         AddError("PeerNetworkMessageHandler.Init", ex);
       }
@@ -68 +63 @@
         sender.Dispose();
         peerListManager.Dispose();
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         AddError("PeerNetworkMessageHandler.Dispose", ex);
       }
     }
 
-    public void PublishDataToNetwork(byte[] data) {
+    public void PublishDataToNetwork(ByteArrayWrapper data) {
       try {
         var allPeers = peerListManager.GetPeerList();
@@ -80 +74 @@
           try {
             sender.SendData(peer, data);
-          }
-          catch (Exception ex) {
+          } catch (Exception ex) {
             AddError("PeerNetworkMessageHandler.PublishDataToNetwork(during sending to one peer!)", ex);
           }
         }
 
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         AddError("PeerNetworkMessageHandler.PublishDataToNetwork", ex);
       }
     }
 
-    public List<KeyValuePair<PeerInfo, byte[]>> GetDataFromNetwork() {
+    public List<KeyValuePair<PeerInfo, ByteArrayWrapper>> GetDataFromNetwork() {
       try {
-        List<KeyValuePair<PeerInfo, byte[]>> res = new List<KeyValuePair<PeerInfo, byte[]>>();
-        KeyValuePair<PeerInfo, byte[]> item;
+        List<KeyValuePair<PeerInfo, ByteArrayWrapper>> res = new List<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
+        KeyValuePair<PeerInfo, ByteArrayWrapper> item;
         lock (activeQueueLocker) {
           //changing the current queue for storing items to send
@@ -111 +103 @@
         }
         return res;//.ToArray();
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         AddError("PeerNetworkMessageHandler.GetDataFromNetwork", ex);
         return null;
@@ -142 +133 @@
           .Select(ip => ip.ToString())
           .First(str => str.StartsWith(ipPrefix));
-      }
-      catch { return null; }
+      } catch { return null; }
     }
 
@@ -149 +139 @@
       if (e != null && e.Sender.ProblemInstance.Equals(ownInstance.ProblemInstance)) {
         lock (activeQueueLocker) {
-          writeQueue.Enqueue(new KeyValuePair<PeerInfo, byte[]>(e.Sender, e.data));
+          writeQueue.Enqueue(new KeyValuePair<PeerInfo, ByteArrayWrapper>(e.Sender, ByteArrayWrapper.CreateByteArrayWrapper(e.data)));
         }
 
@@ -160 +150 @@
         try {
           peerListManager.SendLogToServer(string.Concat("Source: ", source, ", Exception: ", ex.Message));
-        }
-        catch { }
+        } catch { }
       }
     }

branches/thasling/DistributedGA/DistributedGA.Core/Implementation/WcfMessageSender.cs

@@ -1 +1 @@
 using System;
+using System.CodeDom;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Linq;
 using System.ServiceModel;
 using System.Threading.Tasks;
@@ -15 +17 @@
 
     //providing two queues for faster access
-    private SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>> bufferedMessagesWrite;
-    private SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>> bufferedMessagesRead;
+    private SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>> bufferedMessagesWrite;
+    private SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>> bufferedMessagesRead;
 
     private Timer timer; //sends cached messages to network in background
 
     private Object timerLock = new Object();
+    private bool isActive = false;
 
     public void Init(PeerInfo source, int messageCacheCapacity) {
       myself = source;
-      bufferedMessagesRead = new SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>>();
+      bufferedMessagesRead = new SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
       bufferedMessagesRead.Limit = messageCacheCapacity;
-      bufferedMessagesWrite = new SizedConcurrentQueue<KeyValuePair<PeerInfo, byte[]>>();
+      bufferedMessagesWrite = new SizedConcurrentQueue<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
       bufferedMessagesWrite.Limit = bufferedMessagesRead.Limit;
       timer = new Timer(1000 * 10); //each 10 seconds
@@ -33 +36 @@
     }
 
-    public void SendData(PeerInfo destination, byte[] data) {
-      bufferedMessagesWrite.Enqueue(new KeyValuePair<PeerInfo, byte[]>(destination, data));
+    public void SendData(PeerInfo destination, ByteArrayWrapper data) {
+      bufferedMessagesWrite.Enqueue(new KeyValuePair<PeerInfo, ByteArrayWrapper>(destination, data));
     }
 
@@ -44 +47 @@
 
     private void GenerateSendingTasks(object sender, ElapsedEventArgs e) {
+      if (isActive) return;
       lock (timerLock) {
-        //changing the queues...
-        var tmp = bufferedMessagesRead;
-        bufferedMessagesRead = bufferedMessagesWrite;
-        bufferedMessagesWrite = tmp;
-        List<KeyValuePair<PeerInfo, byte[]>> messages = new List<KeyValuePair<PeerInfo, byte[]>>();
-        while (!bufferedMessagesRead.IsEmpty) {
-          KeyValuePair<PeerInfo, byte[]> message;
-          if (bufferedMessagesRead.TryDequeue(out message)) {
-            messages.Add(message);
-            //Task.Factory.StartNew(() => SendDataFromQueue(message.Key, message.Value), TaskCreationOptions.LongRunning);
+        try {
+          isActive = true;
+          //changing the queues...
+          var tmp = bufferedMessagesRead;
+          bufferedMessagesRead = bufferedMessagesWrite;
+          bufferedMessagesWrite = tmp;
+          List<KeyValuePair<PeerInfo, ByteArrayWrapper>> messages = new List<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
+          while (!bufferedMessagesRead.IsEmpty) {
+            KeyValuePair<PeerInfo, ByteArrayWrapper> message;
+            if (bufferedMessagesRead.TryDequeue(out message)) {
+              messages.Add(message);
+              //Task.Factory.StartNew(() => SendDataFromQueue(message.Key, message.Value), TaskCreationOptions.LongRunning);
+            }
           }
-        }
-        //now: merge them and start sending tasks
+          //now: merge them and start sending tasks
 
-        List<KeyValuePair<PeerInfo, byte[][]>> mergedMessages = MergeMessages(messages);
-        foreach (var item in mergedMessages) {
-          Task.Factory.StartNew(() => SendDataFromQueue(item.Key, item.Value), TaskCreationOptions.LongRunning);
+          List<KeyValuePair<PeerInfo, ByteArrayWrapper[]>> mergedMessages = MergeMessages(messages);
+
+          var runningTasks = new List<Task>();
+          foreach (var item in mergedMessages) {
+            var itemToSend = item;
+            var t = Task.Factory.StartNew(() => SendDataFromQueue(itemToSend.Key, itemToSend.Value),
+              TaskCreationOptions.LongRunning);
+            runningTasks.Add(t);
+          }
+
+          Task.WaitAll(runningTasks.ToArray());
+        } finally {
+          isActive = false;
         }
       }
@@ -71 +87 @@
     /// <param name="messages">a list with multiple messages to the same  destination</param>
     /// <returns></returns>
-    private List<KeyValuePair<PeerInfo, byte[][]>> MergeMessages(List<KeyValuePair<PeerInfo, byte[]>> messages) {
-      List<KeyValuePair<PeerInfo, byte[][]>> res = new List<KeyValuePair<PeerInfo, byte[][]>>();
-      Dictionary<PeerInfo, List<byte[]>> cache = new Dictionary<PeerInfo, List<byte[]>>();
+    private List<KeyValuePair<PeerInfo, ByteArrayWrapper[]>> MergeMessages(List<KeyValuePair<PeerInfo, ByteArrayWrapper>> messages) {
+      Dictionary<PeerInfo, List<ByteArrayWrapper>> cache = new Dictionary<PeerInfo, List<ByteArrayWrapper>>();
       foreach (var messagePackage in messages) {
         if (!cache.ContainsKey(messagePackage.Key)) {
-          cache.Add(messagePackage.Key, new List<byte[]>());
+          cache.Add(messagePackage.Key, new List<ByteArrayWrapper>());
         }
         //for (int i = 0; i <= messagePackage.Value.GetUpperBound(0); i++) {
@@ -82 +97 @@
         //}
       }
-      //now we have a dictionary with all messages per destionation
-      //so: create a byte[][] again and return
-      foreach (var dest in cache.Keys) {
-        byte[][] messagesPerDest = new byte[cache[dest].Count][];
-        for (int i = 0; i <= messagesPerDest.GetUpperBound(0); i++) {
-          messagesPerDest[i] = cache[dest][i];
-        }
-        res.Add(new KeyValuePair<PeerInfo, byte[][]>(dest, messagesPerDest));
-      }
-      return res;
+
+      return cache.Keys.Select(dest => new KeyValuePair<PeerInfo, ByteArrayWrapper[]>(dest, cache[dest].ToArray())).ToList();
     }
 
@@ -99 +106 @@
     /// <param name="destination">destination peer</param>
     /// <param name="data">data to send</param>
-    private void SendDataFromQueue(PeerInfo destination, byte[][] data) {
+    private void SendDataFromQueue(PeerInfo destination, ByteArrayWrapper[] data) {
       try {
         Console.WriteLine(string.Format("Sending data to {0}:{1} in the background...", destination.IpAddress, destination.Port));
@@ -117 +124 @@
         using (var myChannelFactory = new ChannelFactory<IMessageContract>(binding, endpoint)) {
           using (IClientChannel client = (IClientChannel)myChannelFactory.CreateChannel()) {
-            for (int i = 0; i < data.GetUpperBound(0) + 1; i++) {
-              ((IMessageContract)client).SendData(myself, data[i]); //maybe exception...
+            for (int i = 0; i < data.GetLength(0); i++) {
+              ((IMessageContract)client).SendData(myself, data[i].Array); //maybe exception...
             }
           }
         }
-      }
-      catch (Exception ex) {
+      } catch (Exception ex) {
         //ignore 
         Console.WriteLine(ex.Message);

branches/thasling/DistributedGA/DistributedGA.Core/Implementation/WcfPeerListManager.cs

@@ -58 +58 @@
     }
 
-    private List<PeerInfo> ChoosePeersForMessaging(ref List<PeerInfo> allPeers) {
+    private List<PeerInfo> ChoosePeersForMessaging(List<PeerInfo> allPeers) {
       Shuffle<PeerInfo>(allPeers);
       int toTake = Convert.ToInt32(allPeers.Count * communicationRate) + 1;
@@ -64 +64 @@
         toTake = 1;
       }
-      return allPeers.Take(toTake).ToList(); ;
+      return allPeers.Take(toTake).ToList();
     }
 
@@ -89 +89 @@
             }
           }
-          cachedPeerList = ChoosePeersForMessaging(ref allPeers);
+          cachedPeerList = ChoosePeersForMessaging(allPeers);
         }
         catch { } //nothing to do

branches/thasling/DistributedGA/DistributedGA.Core/Interface/IMessageHandler.cs

@@ -2 +2 @@
 using System.Collections.Generic;
 using DistributedGA.Core.Domain;
+using DistributedGA.Core.Util;
 
 namespace DistributedGA.Core.Interface {
@@ -9 +10 @@
     void Init(string lanIpPrefix, string contactServerUrl, string problemInstance, int messageCacheCapacty, double communicationRate);
 
-    void PublishDataToNetwork(byte[] data);
+    void PublishDataToNetwork(ByteArrayWrapper data);
 
-    List<KeyValuePair<PeerInfo, byte[]>> GetDataFromNetwork();
+    List<KeyValuePair<PeerInfo, ByteArrayWrapper>> GetDataFromNetwork();
 
     PeerInfo GetPeerInfo();

branches/thasling/DistributedGA/DistributedGA.Core/Interface/IMessageSender.cs

@@ -1 +1 @@
 using DistributedGA.Core.Domain;
+using DistributedGA.Core.Util;
 
 namespace DistributedGA.Core.Interface {
@@ -7 +8 @@
     void Init(PeerInfo source, int messageCacheCapacity);
 
-    void SendData(PeerInfo destination, byte[] data);
+    void SendData(PeerInfo destination, ByteArrayWrapper data);
 
     void Dispose();

branches/thasling/DistributedGA/DistributedGA.Hive/P2PMigrationAnalyzer.cs

@@ -28 +28 @@
 using DistributedGA.Core.Implementation;
 using DistributedGA.Core.Interface;
+using DistributedGA.Core.Util;
 using DistributedGA.Hive;
 using HeuristicLab.Common;
@@ -119 +120 @@
       Parameters.Add(new ValueParameter<StringValue>("JobGUID", "", new StringValue(Guid.NewGuid().ToString())));
       Parameters.Add(new ValueParameter<ILog>("Log", "The log", new Log(1000)));
+      Parameters.Add(new ValueParameter<IntValue>("ByteArraysAllocated", new IntValue(0)));
+      Parameters.Add(new ValueParameter<IntValue>("ByteArraysFreed", new IntValue(0)));
+      Parameters.Add(new ValueParameter<IntValue>("ByteArraysAlive", new IntValue(0)));
 
       var validValues = new ItemSet<EnumValue<MigrationStrategy>>();
@@ -145 +149 @@
       var communicationRate = ((PercentValue)Parameters["CommunicationRate"].ActualValue).Value;
       var messageCacheCapacity = ((IntValue)Parameters["MessageCacheCapacity"].ActualValue).Value;
-      h.Init(lanIpPrefix, contactServerUri, problemInstance, (int)(100 * messageCacheCapacity), (int)(100 * communicationRate));
+      h.Init(lanIpPrefix, contactServerUri, problemInstance, messageCacheCapacity, (int)(100 * communicationRate));
       h.ExceptionOccurend += ExceptionThrown;
     }
@@ -221 +225 @@
               HeuristicLab.Persistence.Default.Xml.XmlGenerator.Serialize(msgScope, stream);
               message = stream.GetBuffer();
-              h.PublishDataToNetwork(message);
+              h.PublishDataToNetwork(ByteArrayWrapper.CreateByteArrayWrapper(message));
             }
           }
@@ -230 +234 @@
           // recieve
           var message = h.GetDataFromNetwork();
-          List<KeyValuePair<PeerInfo, byte[]>> immigrants = new List<KeyValuePair<PeerInfo, byte[]>>();
+          List<KeyValuePair<PeerInfo, ByteArrayWrapper>> immigrants = new List<KeyValuePair<PeerInfo, ByteArrayWrapper>>();
           //limit number of immigrants to use
           if (noOfEmigrants < message.Count) {
@@ -264 +268 @@
           var qualityTranslatedName = QualityParameter.TranslatedName;
           foreach (var msg in immigrants) {
-            using (var stream = new MemoryStream(msg.Value)) {
+            using (var stream = new MemoryStream(msg.Value.Array)) {
               var immigrantScope = HeuristicLab.Persistence.Default.Xml.XmlParser.Deserialize<IScope>(stream);
 
@@ -290 +294 @@
 
       MigrationIterations.Value++;
+
+      ((IValueParameter<IntValue>)Parameters["ByteArraysAllocated"]).Value.Value = ByteArrayWrapper.AllocatedCounter;
+      ((IValueParameter<IntValue>)Parameters["ByteArraysFreed"]).Value.Value = ByteArrayWrapper.FreedCounter;
+      ((IValueParameter<IntValue>)Parameters["ByteArraysAlive"]).Value.Value = ByteArrayWrapper.AliveCounter;
+
       return base.Apply();
     }

branches/thasling/DistributedGA/DistributedGA.sln

@@ -1 +1 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 2013
-VisualStudioVersion = 12.0.21005.1
-MinimumVisualStudioVersion = 10.0.40219.1
+# Visual Studio 2012
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistributedGA.Test", "DistributedGA.Test\DistributedGA.Test.csproj", "{E0E91C06-C56A-454F-9F7C-3FA7AE7F920E}"
 EndProject
@@ -50 +48 @@
     HideSolutionNode = FALSE
   EndGlobalSection
-  GlobalSection(Performance) = preSolution
-    HasPerformanceSessions = true
-  EndGlobalSection
 EndGlobal