Changeset 3913 for trunk/sources/HeuristicLab.Persistence

Timestamp:

06/09/10 16:29:02 (14 years ago)

Author:

epitzer

Message:

Accelerate persistence: (#646)

• dynamically compile field and property accessors
• enable caching of compiled field and property accessors
• delay costly custom attribute retrieval by checking signatures first
• implement custom List<double> serializer
• log deserialization time span

Location:

trunk/sources/HeuristicLab.Persistence/3.3

Files:

• Core/DataMemberAccessor.cs (modified) (7 diffs)
• Core/Serializer.cs (modified) (4 diffs)
• Default/CompositeSerializers/Storable/StorableReflection.cs (modified) (3 diffs)
• Default/CompositeSerializers/Storable/StorableSerializer.cs (modified) (9 diffs)
• Default/Xml/Compact/DoubleList2XmlSerializer.cs (modified) (1 diff)
• Default/Xml/XmlParser.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Persistence/3.3/Core/DataMemberAccessor.cs

@@ -23 +23 @@
 using System.Reflection;
 using HeuristicLab.Persistence.Core;
+using System.Reflection.Emit;
+using System.Collections.Generic;
 
 namespace HeuristicLab.Persistence.Core {
-
+  
   /// <summary>
   /// Encapsulation and abstraction for access a data member of an object
@@ -31 +33 @@
   /// default value and an alternate name can be specified.
   /// </summary>
-  public class DataMemberAccessor {
+  public sealed class DataMemberAccessor {
+
+    #region fields
 
     /// <summary>
     /// The function to get the value of the data member.
     /// </summary>
-    public readonly Func<object> Get;
+    public readonly Func<object, object> Get;
 
     /// <summary>
     /// The function to set the value of the data member.
     /// </summary>
-    public readonly Action<object> Set;
+    public readonly Action<object, object> Set;
 
     /// <summary>
@@ -55 +59 @@
     public readonly object DefaultValue;
 
+    #endregion
+
+    #region constructors
 
     /// <summary>
@@ -63 +70 @@
     /// <param name="name">The name.</param>
     /// <param name="defaultvalue">The defaultvalue.</param>
-    /// <param name="obj">The object.</param>
-    public DataMemberAccessor(MemberInfo memberInfo, string name, object defaultvalue, object obj) {
+    public DataMemberAccessor(MemberInfo memberInfo, string name, object defaultvalue) {
+      Get = GenerateGetter(memberInfo);
+      Set = GenerateSetter(memberInfo);
       Name = name;
       DefaultValue = defaultvalue;
-      if (memberInfo.MemberType == MemberTypes.Field) {
-        FieldInfo fieldInfo = (FieldInfo)memberInfo;
-        Get = () => fieldInfo.GetValue(obj);
-        Set = value => fieldInfo.SetValue(obj, value);
-      } else if (memberInfo.MemberType == MemberTypes.Property) {
-        PropertyInfo propertyInfo = (PropertyInfo)memberInfo;
-        if (!propertyInfo.CanRead || !propertyInfo.CanWrite) {
-          throw new PersistenceException(
-            "Storable properties must implement both a Get and a Set Accessor. ");
-        }
-        Get = () => propertyInfo.GetValue(obj, null);
-        Set = value => propertyInfo.SetValue(obj, value, null);
-      } else {
-        throw new PersistenceException(
-          "The Storable attribute can only be applied to fields and properties.");
-      }
-    }
-
-    /// <summary>
-    /// Wrap existing getter and setter functions.
-    /// </summary>
-    /// <param name="name">The name.</param>
-    /// <param name="defaultValue">The default value.</param>
-    /// <param name="getter">The getter.</param>
-    /// <param name="setter">The setter.</param>
-    public DataMemberAccessor(string name, object defaultValue,
-        Func<object> getter, Action<object> setter) {
-      Name = name;
-      DefaultValue = defaultValue;
-      Get = getter;
-      Set = setter;
     }
 
@@ -103 +80 @@
     /// Create an empty accessor that just encapsulates an object
     /// without access.
-    /// </summary>
-    /// <param name="o">The object</param>
-    public DataMemberAccessor(object o) {
+    /// </summary>    
+    public DataMemberAccessor() {
       Name = null;
       DefaultValue = null;
-      Get = () => o;
-      Set = null;
+      Get = Id;
     }
 
@@ -115 +90 @@
     /// Create an empty accessor that just encapsulates an object
     /// without access.
-    /// </summary>
-    /// <param name="o">The object</param>
+    /// </summary>    
     /// <param name="name">The object's name.</param>
-    public DataMemberAccessor(object o, string name) {
+    public DataMemberAccessor(string name) {
       Name = name;
       DefaultValue = null;
-      Get = () => o;
-      Set = null;
-    }
+      Get = Id;
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DataMemberAccessor"/> class using the
+    /// getter and setter from an exisiting instance but with a new name and default value.
+    /// </summary>
+    /// <param name="dma">The existing DataMemberAccessor.</param>
+    /// <param name="name">The new name.</param>
+    /// <param name="defaultValue">The new default value.</param>
+    public DataMemberAccessor(DataMemberAccessor dma, string name, object defaultValue) {
+      Get = dma.Get;
+      Set = dma.Set;
+      this.Name = name;
+      this.DefaultValue = defaultValue;
+    }
+
+    #endregion
+
+    #region auxiliary methods
 
     /// <summary>
@@ -137 +128 @@
         Get.Method, Set.Method);
     }
+
+    /// <summary>
+    /// The identity function
+    /// </summary>
+    /// <param name="o">An object.</param>
+    /// <returns>its argument o unmodified.</returns>
+    public static object Id(object o) {
+      return o;
+    }
+
+    #endregion
+
+    #region static methods (code generators)
+
+    /// <summary>
+    /// Generate a getter for the given field or property
+    /// </summary>
+    /// <param name="memberInfo">The member info.</param>
+    /// <returns></returns>
+    public static Func<object, object> GenerateGetter(MemberInfo memberInfo) {
+      if (memberInfo.MemberType == MemberTypes.Field) {
+        FieldInfo fieldInfo = (FieldInfo)memberInfo;
+        return GenerateFieldGetter(fieldInfo);
+      } else if (memberInfo.MemberType == MemberTypes.Property) {
+        PropertyInfo propertyInfo = (PropertyInfo)memberInfo;
+        if (!propertyInfo.CanRead || !propertyInfo.CanWrite) {
+          throw new PersistenceException(
+            "Storable properties must implement both a Get and a Set Accessor. ");
+        }
+        return GeneratePropertyGetter(propertyInfo);
+      } else {
+        throw new PersistenceException(
+          "The Storable attribute can only be applied to fields and properties.");
+      }
+    }
+
+    /// <summary>
+    /// Generates a setter for the given field or property.
+    /// </summary>
+    /// <param name="memberInfo">The member info.</param>
+    /// <returns></returns>
+    public static Action<object, object> GenerateSetter(MemberInfo memberInfo) {
+      if (memberInfo.MemberType == MemberTypes.Field) {
+        FieldInfo fieldInfo = (FieldInfo)memberInfo;
+        return GenerateFieldSetter(fieldInfo);
+      } else if (memberInfo.MemberType == MemberTypes.Property) {
+        PropertyInfo propertyInfo = (PropertyInfo)memberInfo;
+        if (!propertyInfo.CanRead || !propertyInfo.CanWrite) {
+          throw new PersistenceException(
+            "Storable properties must implement both a Get and a Set Accessor. ");
+        }
+        return GeneratePropertySetter(propertyInfo);
+      } else {
+        throw new PersistenceException(
+          "The Storable attribute can only be applied to fields and properties.");
+      }
+    }
+
+    /// <summary>
+    /// Generates a dynamically compiled getter to access fields (even private ones).
+    /// </summary>
+    /// <param name="fieldInfo">The field info.</param>
+    /// <returns>A Func&lt;object, object&gt;</returns>
+    public static Func<object, object> GenerateFieldGetter(FieldInfo fieldInfo) {
+      DynamicMethod dm = new DynamicMethod("", typeof(object), new Type[] { typeof(object) }, fieldInfo.DeclaringType, true);
+      ILGenerator ilgen = dm.GetILGenerator();
+      ilgen.Emit(OpCodes.Ldarg_0);
+      ilgen.Emit(OpCodes.Castclass, fieldInfo.DeclaringType);
+      ilgen.Emit(OpCodes.Ldfld, fieldInfo);
+      ilgen.Emit(OpCodes.Box, fieldInfo.FieldType);
+      ilgen.Emit(OpCodes.Ret);
+      return (Func<object, object>)dm.CreateDelegate(typeof(Func<object, object>));
+    }
+
+    /// <summary>
+    /// Generates a dynamically compiled sett to access fields (even private ones).
+    /// </summary>
+    /// <param name="fieldInfo">The field info.</param>
+    /// <returns>An Action&lt;object, object%gt;</returns>
+    public static Action<object, object> GenerateFieldSetter(FieldInfo fieldInfo) {
+      DynamicMethod dm = new DynamicMethod("", null, new Type[] { typeof(object), typeof(object) }, fieldInfo.DeclaringType, true);
+      ILGenerator ilgen = dm.GetILGenerator();
+      ilgen.Emit(OpCodes.Ldarg_0);
+      ilgen.Emit(OpCodes.Castclass, fieldInfo.DeclaringType);
+      ilgen.Emit(OpCodes.Ldarg_1);
+      ilgen.Emit(OpCodes.Unbox_Any, fieldInfo.FieldType);
+      ilgen.Emit(OpCodes.Stfld, fieldInfo);
+      ilgen.Emit(OpCodes.Ret);
+      return (Action<object, object>)dm.CreateDelegate(typeof(Action<object, object>));
+    }
+
+    /// <summary>
+    /// Generates a dynamically compiled getter to access properties (even private ones).
+    /// </summary>
+    /// <param name="propertyInfo">The property info.</param>
+    /// <returns>A Func&lt;object, object&gt;</returns>
+    public static Func<object, object> GeneratePropertyGetter(PropertyInfo propertyInfo) {
+      DynamicMethod dm = new DynamicMethod("", typeof(object), new Type[] { typeof(object) }, propertyInfo.DeclaringType, true);
+      ILGenerator ilgen = dm.GetILGenerator();
+      ilgen.Emit(OpCodes.Ldarg_0);
+      ilgen.Emit(OpCodes.Castclass, propertyInfo.DeclaringType);
+      ilgen.Emit(OpCodes.Callvirt, propertyInfo.GetGetMethod(true));
+      ilgen.Emit(OpCodes.Box, propertyInfo.PropertyType);
+      ilgen.Emit(OpCodes.Ret);
+      return (Func<object, object>)dm.CreateDelegate(typeof(Func<object, object>));
+    }
+
+    /// <summary>
+    /// Generates a dynamically compiled setter to access properties (even private ones).
+    /// </summary>
+    /// <param name="propertyInfo">The property info.</param>
+    /// <returns>An Action&lt;object, object%gt;</returns>
+    public static Action<object, object> GeneratePropertySetter(PropertyInfo propertyInfo) {
+      DynamicMethod dm = new DynamicMethod("", null, new Type[] { typeof(object), typeof(object) }, propertyInfo.DeclaringType, true);
+      ILGenerator ilgen = dm.GetILGenerator();
+      ilgen.Emit(OpCodes.Ldarg_0);
+      ilgen.Emit(OpCodes.Castclass, propertyInfo.DeclaringType);
+      ilgen.Emit(OpCodes.Ldarg_1);
+      ilgen.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType);
+      ilgen.Emit(OpCodes.Callvirt, propertyInfo.GetSetMethod(true));
+      ilgen.Emit(OpCodes.Ret);
+      return (Action<object, object>)dm.CreateDelegate(typeof(Action<object, object>));
+    }
+
+    #endregion
   }
 

trunk/sources/HeuristicLab.Persistence/3.3/Core/Serializer.cs

@@ -187 +187 @@
     /// </returns>
     public IEnumerator<ISerializationToken> GetEnumerator() {
-      var enumerator = Serialize(new DataMemberAccessor(rootName, null, () => obj, null));
+      var enumerator = Serialize(new DataMemberAccessor(rootName), obj);
       if (isTestRun) {
         return AddExceptionCompiler(enumerator);
@@ -206 +206 @@
     private Stack<string> objectGraphTrace = new Stack<string>();
 
-    private IEnumerator<ISerializationToken> Serialize(DataMemberAccessor accessor) {
+    private IEnumerator<ISerializationToken> Serialize(DataMemberAccessor accessor, object obj) {
       
-      object value = accessor.Get();
+      object value = accessor.Get(obj);
       if (value == null)
         return NullReferenceEnumerator(accessor.Name);
@@ -311 +311 @@
       if (metaInfo != null) {
         foreach (var tag in metaInfo) {
-          IEnumerator<ISerializationToken> metaIt = Serialize(new DataMemberAccessor(tag.Value, tag.Name));
+          IEnumerator<ISerializationToken> metaIt = Serialize(new DataMemberAccessor(tag.Name), tag.Value);
           while (metaIt.MoveNext()) {
             if (first) {
@@ -326 +326 @@
       if (tags != null) {
         foreach (var tag in tags) {
-          IEnumerator<ISerializationToken> it = Serialize(new DataMemberAccessor(tag.Value, tag.Name));
+          IEnumerator<ISerializationToken> it = Serialize(new DataMemberAccessor(tag.Name), tag.Value);
           while (it.MoveNext())
             yield return it.Current;

trunk/sources/HeuristicLab.Persistence/3.3/Default/CompositeSerializers/Storable/StorableReflection.cs

@@ -73 +73 @@
 
     private static object[] emptyArgs = new object[0];
+    private static Type[] objectArg = new[] { typeof(object) };
 
     public static IEnumerable<Hook> CollectHooks(HookType hookType, Type type) {
@@ -79 +80 @@
           yield return mi;
       foreach (MemberInfo memberInfo in type.GetMembers(DECLARED_INSTANCE_MEMBERS)) {
+        if (memberInfo.MemberType != MemberTypes.Method)
+          continue;
+        MethodInfo methodInfo = memberInfo as MethodInfo;
+        if (methodInfo.ReturnType != typeof(void))
+          continue;
+        if (methodInfo.GetParameters().Length > 0)
+          continue;
         foreach (StorableHookAttribute hook in memberInfo.GetCustomAttributes(typeof(StorableHookAttribute), false)) {
           if (hook != null && hook.HookType == hookType) {
-            MethodInfo methodInfo = memberInfo as MethodInfo;
-            if (memberInfo.MemberType != MemberTypes.Method || memberInfo == null)
-              throw new ArgumentException("Storable hooks must be methods");            
-            DynamicMethod dm = new DynamicMethod("", null, new[] { typeof(object) }, type);
-            ILGenerator ilgen = dm.GetILGenerator();
-            ilgen.Emit(OpCodes.Ldarg_0);
-            ilgen.Emit(OpCodes.Callvirt, methodInfo);
-            ilgen.Emit(OpCodes.Ret);
-            yield return (Hook)dm.CreateDelegate(typeof(Hook));            
+            yield return new Hook((o) => methodInfo.Invoke(o, emptyArgs));
           }
         }
@@ -99 +99 @@
     private static void AddMarkedMembers(Type type, List<StorableMemberInfo> storableMembers) {
       foreach (MemberInfo memberInfo in type.GetMembers(DECLARED_INSTANCE_MEMBERS)) {
-        foreach (StorableAttribute attribute in memberInfo.GetCustomAttributes(typeof(StorableAttribute), false)) {
-          storableMembers.Add(new StorableMemberInfo(attribute, memberInfo));
+        if (memberInfo.MemberType == MemberTypes.Field ||
+          memberInfo.MemberType == MemberTypes.Property) {
+          foreach (StorableAttribute attribute in memberInfo.GetCustomAttributes(typeof(StorableAttribute), false)) {
+            storableMembers.Add(new StorableMemberInfo(attribute, memberInfo));
+          }
         }
       }

trunk/sources/HeuristicLab.Persistence/3.3/Default/CompositeSerializers/Storable/StorableSerializer.cs

@@ -109 +109 @@
     /// <returns>An enumerable of <see cref="Tag"/>s.</returns>
     public IEnumerable<Tag> Decompose(object obj) {
-      foreach (var accessor in GetStorableAccessors(obj)) {
-        yield return new Tag(accessor.Name, accessor.Get());
+      foreach (var accessor in GetStorableAccessors(obj.GetType())) {
+        yield return new Tag(accessor.Name, accessor.Get(obj));
       }
     }
@@ -142 +142 @@
         memberDict.Add(iter.Current.Name, iter.Current);
       }
-      foreach (var accessor in GetStorableAccessors(instance)) {
+      foreach (var accessor in GetStorableAccessors(instance.GetType())) {
         if (memberDict.ContainsKey(accessor.Name)) {
-          accessor.Set(memberDict[accessor.Name].Value);
+          accessor.Set(instance, memberDict[accessor.Name].Value);
         } else if (accessor.DefaultValue != null) {
-          accessor.Set(accessor.DefaultValue);
+          accessor.Set(instance, accessor.DefaultValue);
         }
       }
@@ -160 +160 @@
 
     private static readonly object[] emptyArgs = new object[] { };
+    private static readonly object[] trueArgs = new object[] { true };
 
     private sealed class HookDesignator {
@@ -171 +172 @@
     }
 
-    private sealed class MemberCache : Dictionary<Type, IEnumerable<StorableMemberInfo>> { }
+    private sealed class AccessorListCache : Dictionary<Type, IEnumerable<DataMemberAccessor>> { }
+    private sealed class AccessorCache : Dictionary<MemberInfo, DataMemberAccessor> { }
 
     #endregion
@@ -177 +179 @@
     #region caches
 
-    private MemberCache storableMemberCache = new MemberCache();
+    private AccessorListCache accessorListCache = new AccessorListCache();
+    private AccessorCache accessorCache = new AccessorCache();
 
     private delegate object Constructor();
@@ -191 +194 @@
     #region attribute access
 
-    private IEnumerable<StorableMemberInfo> GetStorableMembers(Type type) {
-      lock (storableMemberCache) {
-        if (storableMemberCache.ContainsKey(type))
-          return storableMemberCache[type];
-        var storablesMembers = StorableReflection.GenerateStorableMembers(type);
-        storableMemberCache[type] = storablesMembers;
-        return storablesMembers;
+    private IEnumerable<DataMemberAccessor> GetStorableAccessors(Type type) {
+      lock (accessorListCache) {
+        if (accessorListCache.ContainsKey(type))
+          return accessorListCache[type];
+        var storableMembers = StorableReflection
+          .GenerateStorableMembers(type)
+          .Select(mi => GetMemberAccessor(mi));
+        accessorListCache[type] = storableMembers;
+        return storableMembers;
+      }      
+    }
+
+    private DataMemberAccessor GetMemberAccessor(StorableMemberInfo mi) {
+      lock (accessorCache) {
+        if (accessorCache.ContainsKey(mi.MemberInfo))
+          return new DataMemberAccessor(accessorCache[mi.MemberInfo], mi.DisentangledName, mi.DefaultValue);
+        DataMemberAccessor dma = new DataMemberAccessor(mi.MemberInfo, mi.DisentangledName, mi.DefaultValue);
+        accessorCache[mi.MemberInfo] = dma;
+        return dma;
       }
     }
@@ -209 +224 @@
         return c;
       }
-    }
+    }    
 
     private Constructor GetDefaultConstructor(Type type) {
       ConstructorInfo ci = type.GetConstructor(ALL_CONSTRUCTORS, null, Type.EmptyTypes, null);
       if (ci == null)
-        return null;
+        return null;      
       DynamicMethod dm = new DynamicMethod("", typeof(object), null, type);
       ILGenerator ilgen = dm.GetILGenerator();
@@ -227 +242 @@
           if (ci.GetParameters().Length != 1 ||
               ci.GetParameters()[0].ParameterType != typeof(bool))
-            throw new PersistenceException("StorableConstructor must have exactly one argument of type bool");
+            throw new PersistenceException("StorableConstructor must have exactly one argument of type bool");          
           DynamicMethod dm = new DynamicMethod("", typeof(object), null, type);
           ILGenerator ilgen = dm.GetILGenerator();
@@ -237 +252 @@
       }
       return null;
-    }
-
-    private IEnumerable<DataMemberAccessor> GetStorableAccessors(object obj) {
-      return GetStorableMembers(obj.GetType())
-        .Select(mi => new DataMemberAccessor(mi.MemberInfo, mi.DisentangledName, mi.DefaultValue, obj));
-    }
+    }    
 
     private void InvokeHook(HookType hookType, object obj) {

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/Compact/DoubleList2XmlSerializer.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System;
+using System.Linq;
 using HeuristicLab.Persistence.Core;
 using HeuristicLab.Persistence.Default.Xml.Primitive;
+using System.Text;
 
 namespace HeuristicLab.Persistence.Default.Xml.Compact {
 
-  internal sealed class DoubleList2XmlSerializer : NumberEnumeration2XmlSerializerBase<List<double>> {
+  internal sealed class DoubleList2XmlSerializer : CompactXmlSerializerBase<List<double>> {
 
-    protected override void Add(IEnumerable enumeration, object o) {
-      ((List<double>)enumeration).Add((double)o);
+    private static readonly char[] separators = new char[] { ';' };
+
+    public override XmlString Format(List<double> list) {
+      StringBuilder sb = new StringBuilder();
+      foreach (var d in list) {
+        sb.Append(Double2XmlSerializer.FormatG17(d)).Append(';');
+      }
+      return new XmlString(sb.ToString());
     }
 
-    protected override IEnumerable Instantiate() {
-      return new List<double>();
+    public override List<double> Parse(XmlString data) {
+      try {
+        var values = data.Data.Split(separators, StringSplitOptions.RemoveEmptyEntries);
+        List<double> list = new List<double>(values.Length);
+        foreach (var value in values) {
+          list.Add(Double2XmlSerializer.ParseG17(value));
+        }
+        return list;
+      } catch (InvalidCastException e) {
+        throw new PersistenceException("Invalid element data during reconstruction of List<double>.", e);
+      } catch (OverflowException e) {
+        throw new PersistenceException("Overflow during element parsing while trying to reconstruct List<double>.", e);
+      }
     }
-
-    protected override string FormatValue(object o) {
-      return Double2XmlSerializer.FormatG17((double)o);
-    }
-
-    protected override object ParseValue(string o) {
-      return Double2XmlSerializer.ParseG17(o);
-    }
-
   }
 }

trunk/sources/HeuristicLab.Persistence/3.3/Default/Xml/XmlParser.cs

@@ -188 +188 @@
     /// <returns>A fresh object instance</returns>
     public static object Deserialize(string filename) {
-      using (ZipFile file = new ZipFile(filename)) {
-        return Deserialize(file);
+      TimeSpan start = System.Diagnostics.Process.GetCurrentProcess().TotalProcessorTime;
+      try {
+        using (ZipFile file = new ZipFile(filename)) {
+          return Deserialize(file);
+        }
+      } finally {
+        TimeSpan end = System.Diagnostics.Process.GetCurrentProcess().TotalProcessorTime;
+        Tracing.Logger.Info(string.Format(
+          "deserialization of {0} took {1} seconds",
+          filename, (end - start).TotalSeconds));
       }
     }