Changeset 1417 for trunk

Timestamp:

03/25/09 14:48:56 (16 years ago)

Author:

gkronber

Message:

Introduced paging for queries to fix problems with large WCF messages. #257 (CEDMA console doesn't handle communication- and timeout exceptions correctly)

Location:

trunk/sources

Files:

• HeuristicLab.CEDMA.Core/Console.cs (modified) (3 diffs)
• HeuristicLab.CEDMA.Core/DataSet.cs (modified) (1 diff)
• HeuristicLab.CEDMA.Core/DataSetList.cs (modified) (1 diff)
• HeuristicLab.CEDMA.Core/HeuristicLab.CEDMA.Core.csproj (modified) (1 diff)
• HeuristicLab.CEDMA.Core/Results.cs (modified) (3 diffs)
• HeuristicLab.CEDMA.Core/ResultsEntry.cs (modified) (1 diff)
• HeuristicLab.CEDMA.Core/StoreProxy.cs (added)
• HeuristicLab.CEDMA.DB.Interfaces/IStore.cs (modified) (1 diff)
• HeuristicLab.CEDMA.DB/Store.cs (modified) (2 diffs)
• HeuristicLab.CEDMA.Server/DispatcherBase.cs (modified) (1 diff)
• HeuristicLab.CEDMA.Server/Server.cs (modified) (1 diff)

trunk/sources/HeuristicLab.CEDMA.Core/Console.cs

@@ -32 +32 @@
 namespace HeuristicLab.CEDMA.Core {
   public class Console : ItemBase, IEditable {
-    private DataSetList dataSetList;
-    private ChannelFactory<IStore> factory;
-    private IStore store;
     private string serverUri;
     public string ServerUri {
@@ -40 +37 @@
     }
 
+    private DataSetList dataSetList;
     public DataSetList DataSetList {
       get { return dataSetList; }
@@ -72 +70 @@
     #endregion
 
-    #region WCF
-    private void ResetConnection() {
-      NetTcpBinding binding = new NetTcpBinding();
-      binding.ReceiveTimeout = new TimeSpan(1, 0, 0);
-      binding.MaxReceivedMessageSize = 1000000000; // 100Mbytes
-      binding.ReaderQuotas.MaxStringContentLength = 1000000000; // also 100M chars
-      binding.ReaderQuotas.MaxArrayLength = 1000000000; // also 100M elements;
-      factory = new ChannelFactory<IStore>(binding);
-      store = factory.CreateChannel(new EndpointAddress(serverUri));
-      dataSetList.Store = store;
-    }
-    #endregion
-
     internal void Connect(string serverUri) {
-      this.serverUri = serverUri;
-      ResetConnection();
+      DataSetList.Store = new StoreProxy(serverUri);
     }
   }

trunk/sources/HeuristicLab.CEDMA.Core/DataSet.cs

@@ -49 +49 @@
         // lazy loading of problem from DB
         if (problem == null) {
-          //var serializedDataVar = new HeuristicLab.CEDMA.DB.Interfaces.Variable("SerializedData");
-          //var persistedData = Store.Query(
-          //  new Statement[] { new Statement(new Entity(Ontology.CedmaNameSpace + Guid), Ontology.PredicateSerializedData, serializedDataVar)});
           var persistedData = Store.Query(
-            "<" + Ontology.CedmaNameSpace + Guid + "> <" + Ontology.PredicateSerializedData.Uri + "> ?SerializedData .");
+            "<" + Ontology.CedmaNameSpace + Guid + "> <" + Ontology.PredicateSerializedData.Uri + "> ?SerializedData .", 0, 10);
           if (persistedData.Count() == 1) {
             Literal persistedLiteral = (Literal)persistedData.First().Get("SerializedData");

trunk/sources/HeuristicLab.CEDMA.Core/DataSetList.cs

@@ -62 +62 @@
       var bindings = store.Query(
         "?DataSet <"+Ontology.PredicateInstanceOf.Uri+"> <"+Ontology.TypeDataSet.Uri+"> ." + Environment.NewLine +
-        "?DataSet <"+Ontology.PredicateName.Uri+"> ?Name .");
-      //  new Statement[] { 
-      //    new Statement(dataSetVar, Ontology.PredicateInstanceOf, Ontology.TypeDataSet),
-      //    new Statement(dataSetVar, Ontology.PredicateName, dataSetNameVar)
-      //});
+        "?DataSet <"+Ontology.PredicateName.Uri+"> ?Name .",0, 100);
       foreach (var binding in bindings) {
         DataSet d = new DataSet(store, (Entity)binding.Get("DataSet"));

trunk/sources/HeuristicLab.CEDMA.Core/HeuristicLab.CEDMA.Core.csproj

@@ -72 +72 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="StoreProxy.cs" />
     <Compile Include="IResultsViewFactory.cs" />
     <Compile Include="ResultsEntry.cs" />

trunk/sources/HeuristicLab.CEDMA.Core/Results.cs

@@ -37 +37 @@
 namespace HeuristicLab.CEDMA.Core {
   public class Results : ItemBase {
+    private const int PAGE_SIZE = 1000;
     private string[] categoricalVariables = null;
     public string[] CategoricalVariables {
@@ -84 +85 @@
 
     private IEnumerable<ResultsEntry> SelectRows() {
+      int page = 0;
+      int resultsReturned = 0;
+      bool newEntry = false;
       if (store == null) yield break;
       entries = new List<ResultsEntry>();
-      var results = store.Query("<" + dataSetEntity + "> <" + Ontology.PredicateHasModel + "> ?Model ." + Environment.NewLine +
-        "?Model ?Attribute ?Value .")
-        .GroupBy(x => (Entity)x.Get("Model"));
-      foreach (var modelBindings in results) {
-        ResultsEntry entry = new ResultsEntry(modelBindings.Key.Uri);
-        foreach (var binding in modelBindings) {
-          if (binding.Get("Value") is Literal) {
-            string name = ((Entity)binding.Get("Attribute")).Uri.Replace(Ontology.CedmaNameSpace, "");
-            if (entry.Get(name) == null) {
-              object value = ((Literal)binding.Get("Value")).Value;
-              entry.Set(name, value);
+      do {
+        var allBindings = store.Query("<" + dataSetEntity + "> <" + Ontology.PredicateHasModel + "> ?Model ." + Environment.NewLine +
+          "?Model ?Attribute ?Value .", page, PAGE_SIZE);
+        var allModelBindings = allBindings.GroupBy(x => (Entity)x.Get("Model"));
+        resultsReturned = allBindings.Count;
+
+        foreach (var modelBindings in allModelBindings) {
+          ResultsEntry entry = entries.FirstOrDefault(x => x.Uri == modelBindings.Key.Uri);
+          newEntry = false;
+          if (entry == null) {
+            entry = new ResultsEntry();
+            entry.Uri = modelBindings.Key.Uri;
+            entries.Add(entry);
+            newEntry = true;
+          }
+          foreach (var binding in modelBindings) {
+            if (binding.Get("Value") is Literal) {
+              string name = ((Entity)binding.Get("Attribute")).Uri.Replace(Ontology.CedmaNameSpace, "");
+              if (entry.Get(name) == null) {
+                object value = ((Literal)binding.Get("Value")).Value;
+                entry.Set(name, value);
+              }
             }
           }
+          if (newEntry) yield return entry;
         }
-        entries.Add(entry);
-        yield return entry;
-      }
+        page++;
+      } while (resultsReturned == PAGE_SIZE);
 
       FireChanged();
@@ -114 +129 @@
     private void LoadModelAttributes() {
       this.ordinalVariables =
-        store.Query("?ModelAttribute <" + Ontology.PredicateInstanceOf + "> <" + Ontology.TypeOrdinalAttribute + "> .")
+        store.Query("?ModelAttribute <" + Ontology.PredicateInstanceOf + "> <" + Ontology.TypeOrdinalAttribute + "> .", 0, 100)
         .Select(s => ((Entity)s.Get("ModelAttribute")).Uri.Replace(Ontology.CedmaNameSpace, ""))
         .ToArray();
       this.categoricalVariables =
-        store.Query("?ModelAttribute <" + Ontology.PredicateInstanceOf + "> <" + Ontology.TypeCategoricalAttribute + "> .")
+        store.Query("?ModelAttribute <" + Ontology.PredicateInstanceOf + "> <" + Ontology.TypeCategoricalAttribute + "> .", 0, 100)
         .Select(s => ((Entity)s.Get("ModelAttribute")).Uri.Replace(Ontology.CedmaNameSpace, ""))
         .ToArray();

trunk/sources/HeuristicLab.CEDMA.Core/ResultsEntry.cs

@@ -43 +43 @@
 
     private string uri;
-    public string Uri { get { return uri; } }
-
-    public ResultsEntry(string uri) {
-      this.uri = uri;
+    public string Uri { 
+      get { return uri; }
+      set { uri = value; }
     }
 

trunk/sources/HeuristicLab.CEDMA.DB.Interfaces/IStore.cs

@@ -35 +35 @@
 
     [OperationContract]
-    ICollection<VariableBindings> Query(ICollection<Statement> query);
+    ICollection<VariableBindings> Query(ICollection<Statement> query, int page, int pageSize);
 
     [OperationContract(Name="QueryN3")]
-    ICollection<VariableBindings> Query(string query);
+    ICollection<VariableBindings> Query(string query, int page, int pageSize);
   }
 }

trunk/sources/HeuristicLab.CEDMA.DB/Store.cs

@@ -62 +62 @@
 
 
-    public ICollection<VariableBindings> Query(string query) {
+    public ICollection<VariableBindings> Query(string query, int page, int pageSize) {
       MyQueryResultSink resultSink = new MyQueryResultSink();
       SemWeb.N3Reader n3Reader = new SemWeb.N3Reader(new StringReader(query));
       SemWeb.Query.GraphMatch matcher = new SemWeb.Query.GraphMatch(n3Reader);
       matcher.Run(store, resultSink);
-      return resultSink.Bindings;
-    }
-
-    public ICollection<VariableBindings> Query(ICollection<Statement> query) {
+      return resultSink.Bindings.Skip(page*pageSize).Take(pageSize).ToList();
+    }
+
+    public ICollection<VariableBindings> Query(ICollection<Statement> query, int page, int pageSize) {
       MyQueryResultSink resultSink = new MyQueryResultSink();
       Translate(query).Run(store, resultSink);
-      return resultSink.Bindings;
+      return resultSink.Bindings.Skip(page * pageSize).Take(pageSize).ToList();
     }
 
@@ -82 +82 @@
       return new SemWeb.Query.GraphMatch(queryStore);
     }
-
-
-    //public ICollection<Statement> Select(Statement template) {
-    //  SemWeb.MemoryStore memStore = new SemWeb.MemoryStore();
-    //  lock (bigLock) {
-    //    store.Select(Translate(template), memStore);
-    //  }
-    //  return memStore.Select(x=>Translate(x)).ToList();
-    //}
-
-    //public ICollection<Statement> Select(SelectFilter filter) {
-    //  SemWeb.MemoryStore memStore = new SemWeb.MemoryStore();
-    //  lock (bigLock) {
-    //    store.Select(Translate(filter), memStore);
-    //  }
-    //  return memStore.Select(x => Translate(x)).ToList();
-    //}
-
-    //private SemWeb.SelectFilter Translate(SelectFilter filter) {
-    //  SemWeb.SelectFilter f = new SemWeb.SelectFilter();
-    //  f.Subjects = Array.ConvertAll(filter.Subjects, s => Translate(s));
-    //  f.Predicates = Array.ConvertAll(filter.Predicates, p => Translate(p));
-    //  f.Objects = Array.ConvertAll(filter.Properties, prop => Translate(prop));
-    //  return f;
-    //}
 
     private static SemWeb.Entity Translate(Entity e) {

trunk/sources/HeuristicLab.CEDMA.Server/DispatcherBase.cs

@@ -68 +68 @@
       };
 
-      Entity[] datasets = store.Query("?DataSet <" + Ontology.PredicateInstanceOf.Uri + "> <" + Ontology.TypeDataSet.Uri + "> .")
+      Entity[] datasets = store.Query("?DataSet <" + Ontology.PredicateInstanceOf.Uri + "> <" + Ontology.TypeDataSet.Uri + "> .", 0, 100)
         .Select(x => (Entity)x.Get("DataSet"))
         .ToArray();

trunk/sources/HeuristicLab.CEDMA.Server/Server.cs

@@ -71 +71 @@
       try {
         NetTcpBinding binding = new NetTcpBinding();
-        binding.SendTimeout = new TimeSpan(1, 0, 0);
-        binding.MaxReceivedMessageSize = 1000000000; // 100Mbytes
-        binding.ReaderQuotas.MaxStringContentLength = 1000000000; // also 100M chars
-        binding.ReaderQuotas.MaxArrayLength = 1000000000; // also 100M elements;
+        binding.SendTimeout = new TimeSpan(10, 0, 0);
+        binding.MaxReceivedMessageSize = int.MaxValue;
+        binding.ReaderQuotas.MaxStringContentLength = int.MaxValue;
+        binding.ReaderQuotas.MaxArrayLength = int.MaxValue;
         host.AddServiceEndpoint(typeof(IStore), binding, cedmaServiceUrl);
         host.Open();