Changeset 13809 for branches

Timestamp:

04/28/16 13:08:54 (9 years ago)

Author:

abeham

Message:

#2457: added file system based cache of runs

File:

• branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs (modified) (4 diffs)

branches/PerformanceComparison/HeuristicLab.OptimizationExpertSystem.Common/3.3/KnowledgeCenter.cs

@@ -39 +39 @@
 using System.Threading;
 using System.Threading.Tasks;
+using Algorithm = HeuristicLab.Clients.OKB.Administration.Algorithm;
+using Problem = HeuristicLab.Clients.OKB.Administration.Problem;
 using RunCreationClient = HeuristicLab.Clients.OKB.RunCreation.RunCreationClient;
 using SingleObjectiveOKBProblem = HeuristicLab.Clients.OKB.RunCreation.SingleObjectiveOKBProblem;
@@ -414 +416 @@
 
         problemId2ProblemInstanceMapping.Clear();
-        progress.Status = "Downloading problem instances...";
+        progress.Status = "Downloading algorithm and problem instances...";
         progress.ProgressValue = 0;
+
         int[] p = { 0 };
         ProblemInstances.UpdateOfRunsInProgress = true;
         ProblemInstances.Clear();
-        var characteristics = new HashSet<string>();
-        var totalProblems = adminClient.Problems.Count(x => x.ProblemClassId == probClassId);
-        Parallel.ForEach(adminClient.Problems.Where(x => x.ProblemClassId == probClassId), new ParallelOptions { MaxDegreeOfParallelism = 8 }, (pInst) => { 
-          var charas = new List<string>();
-          IRun probRun = null;
-          var data = Clients.OKB.Administration.AdministrationClient.GetProblemData(pInst.Id);
-          if (data != null) {
-            using (var stream = new MemoryStream(data)) {
-              try {
-                var prob = (IProblem)XmlParser.Deserialize<IContent>(stream);
-                probRun = new Run() { Name = prob.Name };
-                prob.CollectParameterValues(probRun.Parameters);
-                probRun.Parameters["Problem Name"] = new StringValue(prob.Name);
-                probRun.Parameters["Problem Type"] = new StringValue(prob.GetType().Name);
-                foreach (var v in RunCreationClient.Instance.GetCharacteristicValues(pInst.Id)) {
-                  probRun.Results.Add("Characteristic." + v.Name, RunCreationClient.Instance.ConvertToItem(v));
-                  charas.Add("Characteristic." + v.Name);
-                }
-              } catch { }
-              stream.Close();
-            }
-            if (probRun != null) {
-              lock (characteristics) {
-                problemId2ProblemInstanceMapping.Add(pInst.Id, probRun);
-                ProblemInstances.Add(probRun);
-                progress.Status = string.Format("Downloaded problem {0} (okb-id: {1})....", pInst.Name, pInst.Id);
-                p[0]++;
-                progress.ProgressValue = p[0] / (double)totalProblems;
-                foreach (var c in charas) characteristics.Add(c);
-              }
-            }
-          }
-        });
-        
         algorithmId2AlgorithmInstanceMapping.Clear();
         algorithmId2RunMapping.Clear();
         algorithmInstances.Clear();
-        progress.Status = "Downloading algorithm instances...";
-        progress.ProgressValue = 0;
-        p[0] = 0;
-        Parallel.ForEach(adminClient.Algorithms, new ParallelOptions { MaxDegreeOfParallelism = 8 }, (algInst) => {
-          IAlgorithm alg = null;
-          var data = Clients.OKB.Administration.AdministrationClient.GetAlgorithmData(algInst.Id);
-          if (data != null) {
-            using (var stream = new MemoryStream(data)) {
-              try {
-                alg = (IAlgorithm)XmlParser.Deserialize<IContent>(stream);
-              } catch { }
-              stream.Close();
-            }
-            if (alg != null) {
-              lock (algorithmId2AlgorithmInstanceMapping) {
-                algorithmInstances.Add(alg);
-                algorithmId2AlgorithmInstanceMapping.Add(algInst.Id, alg);
-                progress.Status = string.Format("Downloaded algorithm {0} (okb-id: {1})...", algInst.Name, algInst.Id);
-                p[0]++;
-                progress.ProgressValue = p[0] / (double)adminClient.Algorithms.Count;
-              }
-            }
+
+        var characteristics = new HashSet<string>();
+        var totalProblems = adminClient.Problems.Count(x => x.ProblemClassId == probClassId);
+        var totalAlgorithms = adminClient.Algorithms.Count;
+        var problems = adminClient.Problems.Where(x => x.ProblemClassId == probClassId);
+        var algorithms = adminClient.Algorithms;
+        var combined = problems.Cast<object>().Concat(algorithms.Cast<object>()).Shuffle(new MersenneTwister());
+        Parallel.ForEach(combined, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, (inst) => {
+          var pInst = inst as Clients.OKB.Administration.Problem;
+          if (pInst != null) DownloadProblemInstance(progress, pInst, p, totalProblems + totalAlgorithms, characteristics);
+          else {
+            var aInst = inst as Clients.OKB.Administration.Algorithm;
+            DownloadAlgorithmInstance(progress, aInst, p, totalProblems + totalAlgorithms);
           }
         });
@@ -490 +450 @@
         
         var runList = new List<IRun>();
-        var runIds = queryClient.GetRunIds(problemClassFilter).ToList();
+        var runIds = LoadRunsFromCache(queryClient.GetRunIds(problemClassFilter), runList, progress);
         var batches = runIds.Select((v, i) => new { Idx = i, Val = v }).GroupBy(x => x.Idx / 500, x => x.Val);
-        Parallel.ForEach(batches.Select(x => x.ToList()), new ParallelOptions { MaxDegreeOfParallelism = 4 }, (batch) => {
+        Parallel.ForEach(batches.Select(x => x.ToList()), new ParallelOptions { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 4) },
+          (batch) => {
           var okbRuns = queryClient.GetRunsWithValues(batch, true, interestingValues);
-          var hlRuns = okbRuns.AsParallel().Select(x => new { AlgorithmId = x.Algorithm.Id, Run = queryClient.ConvertToOptimizationRun(x) }).ToList();
+          var hlRuns = okbRuns.AsParallel().Select(x => new { AlgorithmId = x.Algorithm.Id, RunId = x.Id, Run = queryClient.ConvertToOptimizationRun(x) }).ToList();
           lock (runList) {
+            var toCache = new List<Tuple<long, long, IRun>>();
             foreach (var r in hlRuns) {
               algorithmId2RunMapping.Add(r.AlgorithmId, r.Run);
               runList.Add(r.Run);
-            }
+              toCache.Add(Tuple.Create(r.AlgorithmId, r.RunId, r.Run));
+            }
+            SaveToCache(toCache);
             progress.Status = string.Format("Downloaded runs {0} to {1} of {2}...", p[0], p[0] + batch.Count, count);
             p[0] += batch.Count;
@@ -558 +522 @@
       } finally { progress.Finish(); ProblemInstances.UpdateOfRunsInProgress = false; }
       UpdateInstanceProjection(ProblemInstances.ResultNames.Where(x => x.StartsWith("Characteristic.")).ToArray());
+    }
+
+    private void DownloadAlgorithmInstance(IProgress progress, Algorithm algInst, int[] p, int total) {
+      IAlgorithm alg = null;
+      var data = Clients.OKB.Administration.AdministrationClient.GetAlgorithmData(algInst.Id);
+      if (data != null) {
+        using (var stream = new MemoryStream(data)) {
+          try {
+            alg = (IAlgorithm)XmlParser.Deserialize<IContent>(stream);
+          } catch { }
+          stream.Close();
+        }
+        if (alg != null) {
+          lock (progress) {
+            algorithmInstances.Add(alg);
+            algorithmId2AlgorithmInstanceMapping.Add(algInst.Id, alg);
+            progress.Status = string.Format("Downloaded algorithm {0} (okb-id: {1})...", algInst.Name, algInst.Id);
+            p[0]++;
+            progress.ProgressValue = p[0] / (double)total;
+          }
+        }
+      }
+    }
+
+    private void DownloadProblemInstance(IProgress progress, Problem pInst, int[] p, int totalProblems, HashSet<string> characteristics) {
+      var charas = new List<string>();
+      IRun probRun = null;
+      var data = Clients.OKB.Administration.AdministrationClient.GetProblemData(pInst.Id);
+      if (data != null) {
+        using (var stream = new MemoryStream(data)) {
+          try {
+            var prob = (IProblem)XmlParser.Deserialize<IContent>(stream);
+            probRun = new Run() {Name = prob.Name};
+            prob.CollectParameterValues(probRun.Parameters);
+            probRun.Parameters["Problem Name"] = new StringValue(prob.Name);
+            probRun.Parameters["Problem Type"] = new StringValue(prob.GetType().Name);
+            foreach (var v in RunCreationClient.Instance.GetCharacteristicValues(pInst.Id)) {
+              probRun.Results.Add("Characteristic." + v.Name, RunCreationClient.Instance.ConvertToItem(v));
+              charas.Add("Characteristic." + v.Name);
+            }
+          } catch { }
+          stream.Close();
+        }
+        if (probRun != null) {
+          lock (progress) {
+            problemId2ProblemInstanceMapping.Add(pInst.Id, probRun);
+            ProblemInstances.Add(probRun);
+            progress.Status = string.Format("Downloaded problem {0} (okb-id: {1})....", pInst.Name, pInst.Id);
+            p[0]++;
+            progress.ProgressValue = p[0] / (double)totalProblems;
+            foreach (var c in charas) characteristics.Add(c);
+          }
+        }
+      }
+    }
+
+    private List<long> LoadRunsFromCache(IEnumerable<long> runIds, List<IRun> runList, IProgress progress) {
+      var hashSet = new HashSet<long>(runIds);
+      var total = hashSet.Count;
+      try {
+        var path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HeuristicLab.OKB", "cache", "runs");
+        Parallel.ForEach(Directory.EnumerateDirectories(path).Select((d, i) => new { Index = i, Directory = d }).GroupBy(x => x.Index / 100), new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount },
+          (folderGroup) => {
+          var localRunList = new List<Tuple<long, long, IRun>>();
+          foreach (var runPath in folderGroup.Select(x => x.Directory)) {
+            long runId;
+            var runFolder = new DirectoryInfo(runPath).Name;
+            if (!long.TryParse(runFolder, out runId) || !hashSet.Contains(runId)) continue;
+            var runFilePath = Directory.EnumerateFiles(runPath).Single();
+            var runFileName = Path.GetFileNameWithoutExtension(runFilePath);
+            long algId;
+            if (!long.TryParse(runFileName, out algId)) continue;
+            IRun run = null;
+            try {
+              using (var file = File.OpenRead(runFilePath))
+                run = XmlParser.Deserialize<IRun>(file);
+            } catch {
+              File.Delete(runFilePath);
+              Directory.Delete(runPath);
+            }
+            if (run != null) localRunList.Add(Tuple.Create(algId, runId, run));
+          }
+          lock (runList) {
+            foreach (var r in localRunList) {
+              hashSet.Remove(r.Item2);
+              algorithmId2RunMapping.Add(r.Item1, r.Item3);
+              runList.Add(r.Item3);
+            }
+            progress.Status = string.Format("Retrieved {0} of {1} from cache", runList.Count, total);
+            progress.ProgressValue = (double)runList.Count / total;
+          }
+        });
+      } catch { }
+      return hashSet.ToList();
+    }
+
+    private void SaveToCache(IEnumerable<Tuple<long, long, IRun>> runs) {
+      try {
+        var path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HeuristicLab.OKB", "cache", "runs");
+        if (!Directory.Exists(path)) Directory.CreateDirectory(path);
+        foreach (var r in runs) {
+          var runPath = Path.Combine(path, r.Item2.ToString());
+          if (!Directory.Exists(runPath)) Directory.CreateDirectory(runPath);
+          using (var file = File.Open(Path.Combine(runPath, r.Item1.ToString()), FileMode.Create, FileAccess.ReadWrite)) {
+            XmlGenerator.Serialize(r.Item3, file);
+          }
+        }
+      } catch { }
     }