Changeset 17005

Timestamp:

06/11/19 20:59:44 (5 years ago)

Author:

gkronber

Message:

#2994: merged r16944:16997 from trunk to branch

Location:

branches/2994-AutoDiffForIntervals

Files:

• . (modified) (1 prop)
• HeuristicLab.Common/3.3/Content/ContentManager.cs (modified) (2 diffs)
• HeuristicLab.Core (modified) (1 prop)
• HeuristicLab.Core.Views (modified) (1 prop)
• HeuristicLab.Core.Views/3.3/Clipboard.cs (modified) (3 diffs)
• HeuristicLab.Core.Views/3.3/HeuristicLab.Core.Views-3.3.csproj (modified) (1 diff)
• HeuristicLab.Core.Views/3.3/Plugin.cs.frame (modified) (1 diff)
• HeuristicLab.Core/3.3/PersistenceContentManager.cs (modified) (2 diffs)
• HeuristicLab.DataPreprocessing.Views (modified) (1 prop)
• HeuristicLab.DataPreprocessing.Views/3.4/DataGridContentView.cs (modified) (1 diff)
• HeuristicLab.ExtLibs (modified) (1 prop)
• HeuristicLab.ExtLibs/HeuristicLab.OrTools/7.0.0/HeuristicLab.OrTools-7.0.0/HeuristicLab.OrTools-7.0.0.csproj (modified) (4 diffs)
• HeuristicLab.Optimizer (modified) (1 prop)
• HeuristicLab.Optimizer/3.3/FileManager.cs (modified) (2 diffs)
• HeuristicLab.Optimizer/3.3/HeuristicLab.Optimizer-3.3.csproj (modified) (1 diff)
• HeuristicLab.Optimizer/3.3/Plugin.cs.frame (modified) (1 diff)
• HeuristicLab.Optimizer/3.3/StartPage.cs (modified) (2 diffs)
• HeuristicLab.Persistence (modified) (1 prop)
• HeuristicLab.Persistence/3.3/Default/Xml/XmlParser.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4 (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/MultiObjective/PearsonRSquaredAverageSimilarityEvaluator.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/SymbolicDataAnalysisBuildingBlockAnalyzer.cs (deleted)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Crossovers/SymbolicDataAnalysisExpressionDiversityPreservingCrossover.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashExtensions.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/SymbolicExpressionTreeHash.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisExpressionTreeAverageSimilarityCalculator.cs (copied) (copied from trunk/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisExpressionTreeAverageSimilarityCalculator.cs)
• HeuristicLab.Tests (modified) (1 prop)
• HeuristicLab.Tests/HeuristicLab.Persistence.Attic/PersistenceConsistencyChecks.cs (modified) (1 diff)

branches/2994-AutoDiffForIntervals/HeuristicLab.Common/3.3/Content/ContentManager.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
+using HEAL.Attic;
 
 namespace HeuristicLab.Common {
   public abstract class ContentManager {
+    public class Info {
+      public Info() { }
+
+      public Info(string filename, TimeSpan duration) {
+        Filename = filename;
+        Duration = duration;
+      }
+
+      public Info(string filename, SerializationInfo serInfo) {
+        Filename = filename;
+        Duration = serInfo.Duration;
+        UnknownTypeGuids = serInfo.UnknownTypeGuids;
+        NumberOfSerializedObjects = serInfo.NumberOfSerializedObjects;
+        SerializedTypes = serInfo.SerializedTypes;        
+      }
+
+      public TimeSpan Duration { get; internal set; }
+      public IEnumerable<Guid> UnknownTypeGuids { get; internal set; } = Enumerable.Empty<Guid>();
+      public int NumberOfSerializedObjects { get; internal set; }
+      public IEnumerable<Type> SerializedTypes { get; internal set; } = Enumerable.Empty<Type>();
+      public string Filename { get; internal set; } = string.Empty;
+    }
+
     private static ContentManager instance;
 
@@ -36 +63 @@
 
     public static IStorableContent Load(string filename) {
+      return Load(filename, out var _);
+    }
+
+    public static IStorableContent Load(string filename, out Info serializationInfo) {
       if (instance == null) throw new InvalidOperationException("ContentManager is not initialized.");
-      IStorableContent content = instance.LoadContent(filename);
-      content.Filename = filename;
+      IStorableContent content = instance.LoadContent(filename, out serializationInfo);
+      
+      if (content != null) content.Filename = filename;
       return content;
     }
-    public static void LoadAsync(string filename, Action<IStorableContent, Exception> loadingCompletedCallback) {
+
+
+    public static Task LoadAsync(string filename, Action<IStorableContent, Exception> loadingCompletedCallback) {
+      return LoadAsync(filename, (content, error, info) => loadingCompletedCallback(content, error)); // drop info
+    }
+
+    public static async Task LoadAsync(string filename, Action<IStorableContent, Exception, Info> loadingCompletedCallback) {
       if (instance == null) throw new InvalidOperationException("ContentManager is not initialized.");
-      var func = new Func<string, IStorableContent>(instance.LoadContent);
-      func.BeginInvoke(filename, delegate (IAsyncResult result) {
-        Exception error = null;
-        IStorableContent content = null;
-        try {
-          content = func.EndInvoke(result);
-          content.Filename = filename;
-        } catch (Exception ex) {
-          error = ex;
-        }
-        loadingCompletedCallback(content, error);
-      }, null);
+
+      Exception error = null;
+      IStorableContent result = null;
+      Info serializationInfo = null;
+      try {
+        result = await Task.Run(() => {
+          var content = instance.LoadContent(filename, out serializationInfo);
+          if(content!=null) content.Filename = filename;
+          return content;
+        });
+      } catch(Exception ex) {
+        error = ex;
+      }
+      loadingCompletedCallback(result, error, serializationInfo);
     }
-    protected abstract IStorableContent LoadContent(string filename);
+
+    protected abstract IStorableContent LoadContent(string filename, out Info serializationInfo);
 
     public static void Save(IStorableContent content, string filename, bool compressed, CancellationToken cancellationToken = default(CancellationToken)) {

branches/2994-AutoDiffForIntervals/HeuristicLab.Core.Views/3.3/Clipboard.cs

@@ -24 +24 @@
 using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
 using System.Threading;
@@ -31 +30 @@
 using HeuristicLab.Common;
 using HeuristicLab.MainForm;
-using HeuristicLab.Persistence.Default.Xml;
 using HeuristicLab.PluginInfrastructure;
 
@@ -157 +155 @@
       foreach (string filename in items) {
         try {
-          var ser = new ProtoBufSerializer();
-          T item = (T)ser.Deserialize(filename);
+          T item = (T)ContentManager.Load(filename);
           OnItemLoaded(item, progressBar.Maximum / items.Length);
-        } catch (Exception) {
-          try {
-            // try old format if protobuf deserialization fails
-            T item = XmlParser.Deserialize<T>(filename);
-            OnItemLoaded(item, progressBar.Maximum / items.Length);
-          } catch (Exception) { }
+        } catch(Exception) {
+          // ignore if loading a clipboad item fails.
         }
       }

branches/2994-AutoDiffForIntervals/HeuristicLab.Core.Views/3.3/HeuristicLab.Core.Views-3.3.csproj

@@ -375 +375 @@
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
-      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
-      <Name>HeuristicLab.Persistence-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
       <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>

branches/2994-AutoDiffForIntervals/HeuristicLab.Core.Views/3.3/Plugin.cs.frame

@@ -35 +35 @@
   [PluginDependency("HeuristicLab.MainForm", "3.3")]
   [PluginDependency("HeuristicLab.MainForm.WindowsForms", "3.3")]
-  [PluginDependency("HeuristicLab.Persistence", "3.3")]
   public class HeuristicLabCoreViewsPlugin : PluginBase {
   }

branches/2994-AutoDiffForIntervals/HeuristicLab.Core/3.3/PersistenceContentManager.cs

@@ -25 +25 @@
 using HEAL.Attic;
 using System;
+using System.Diagnostics;
 
 namespace HeuristicLab.Core {
@@ -30 +31 @@
     public PersistenceContentManager() : base() { }
 
-    protected override IStorableContent LoadContent(string filename) {
-      // first try to load using the new persistence format
-      var ser = new ProtoBufSerializer();
-      try {
-        return (IStorableContent)ser.Deserialize(filename, out SerializationInfo info);
-      } catch (Exception e) {
-        try {
-          // try old format if new format fails
-          return XmlParser.Deserialize<IStorableContent>(filename);
-        } catch (Exception e2) {
-          throw new AggregateException($"Cannot open file {filename}", e, e2);
-        }
+    protected override IStorableContent LoadContent(string filename, out Info info) {
+      bool useOldPersistence = XmlParser.CanOpen(filename);
+      IStorableContent content = null;
+      if (useOldPersistence) {
+        var sw = new Stopwatch();
+        sw.Start();
+        content = XmlParser.Deserialize<IStorableContent>(filename);
+        sw.Stop();
+        info = new Info(filename, sw.Elapsed);
+      } else {
+        var ser = new ProtoBufSerializer();
+        content = (IStorableContent)ser.Deserialize(filename, out SerializationInfo serInfo);
+        info = new Info(filename, serInfo);
       }
+      if (content == null) throw new PersistenceException($"Cannot deserialize root element of {filename}");
+      return content;
     }
+
 
     protected override void SaveContent(IStorableContent content, string filename, bool compressed, CancellationToken cancellationToken) {

branches/2994-AutoDiffForIntervals/HeuristicLab.DataPreprocessing.Views/3.4/DataGridContentView.cs

@@ -411 +411 @@
         var foundIndices = new List<int>();
         for (int idx = 0; idx < filteredRows.Length; ++idx) {
-          var notFilteredThusFound = !filteredRows[idx];
-          if (notFilteredThusFound) {
+          if (filteredRows[idx]) {
             foundIndices.Add(idx);
           }

branches/2994-AutoDiffForIntervals/HeuristicLab.ExtLibs/HeuristicLab.OrTools/7.0.0/HeuristicLab.OrTools-7.0.0/HeuristicLab.OrTools-7.0.0.csproj

@@ -43 +43 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
     <DebugSymbols>true</DebugSymbols>
-    <OutputPath>bin\x64\Debug\</OutputPath>
+    <OutputPath>..\..\..\..\bin\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -52 +52 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
-    <OutputPath>bin\x64\Release\</OutputPath>
+    <OutputPath>..\..\..\..\bin\</OutputPath>
     <DefineConstants>TRACE</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -63 +63 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
     <DebugSymbols>true</DebugSymbols>
-    <OutputPath>bin\x86\Debug\</OutputPath>
+    <OutputPath>..\..\..\..\bin\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -72 +72 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
-    <OutputPath>bin\x86\Release\</OutputPath>
+    <OutputPath>..\..\..\..\bin\</OutputPath>
     <DefineConstants>TRACE</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>

branches/2994-AutoDiffForIntervals/HeuristicLab.Optimizer/3.3/FileManager.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Windows.Forms;
@@ -73 +74 @@
     }
 
-    private static void LoadingCompleted(IStorableContent content, Exception error) {
+    private static void LoadingCompleted(IStorableContent content, Exception error, ContentManager.Info info) {
       try {
         if (error != null) throw error;
+        if (info!=null && info.UnknownTypeGuids.Any()) {
+          var message = "Unknown type guids: " + string.Join(Environment.NewLine, info.UnknownTypeGuids);
+          MessageBox.Show((Control)MainFormManager.MainForm, message, $"File {info.Filename} not restored completely", MessageBoxButtons.OK, MessageBoxIcon.Information);
+        }
         IView view = MainFormManager.MainForm.ShowContent(content);
         if (view == null)

branches/2994-AutoDiffForIntervals/HeuristicLab.Optimizer/3.3/HeuristicLab.Optimizer-3.3.csproj

@@ -312 +312 @@
       <Private>False</Private>
     </ProjectReference>
-    <ProjectReference Include="..\..\HeuristicLab.Persistence\3.3\HeuristicLab.Persistence-3.3.csproj">
-      <Project>{102BC7D3-0EF9-439C-8F6D-96FF0FDB8E1B}</Project>
-      <Name>HeuristicLab.Persistence-3.3</Name>
-      <Private>False</Private>
-    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
       <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>

branches/2994-AutoDiffForIntervals/HeuristicLab.Optimizer/3.3/Plugin.cs.frame

@@ -43 +43 @@
   [PluginDependency("HeuristicLab.Optimization", "3.3")]
   [PluginDependency("HeuristicLab.Parameters", "3.3")]
-  [PluginDependency("HeuristicLab.Persistence", "3.3")]
   [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
   public class HeuristicLabOptimizerPlugin : PluginBase {

branches/2994-AutoDiffForIntervals/HeuristicLab.Optimizer/3.3/StartPage.cs

@@ -30 +30 @@
 using HeuristicLab.Core;
 using HeuristicLab.MainForm;
-using HeuristicLab.Persistence.Default.Xml;
 
 namespace HeuristicLab.Optimizer {
@@ -121 +120 @@
         using (var stream = assembly.GetManifestResourceStream(name)) {
           WriteStreamToTempFile(stream, path); // create a file in a temporary folder (persistence cannot load these files directly from the stream)
-          var item = XmlParser.Deserialize<INamedItem>(path);
+          var item = (INamedItem)ContentManager.Load(path);
           OnSampleLoaded(item, group, 1.0 / count);
         }

branches/2994-AutoDiffForIntervals/HeuristicLab.Persistence/3.3/Default/Xml/XmlParser.cs

@@ -288 +288 @@
       }
     }
+
+    /// <summary>
+    /// Checks if the given file can be opened as <see cref="ZipArchive" />.
+    /// </summary>
+    /// <param name="filename">The filename.</param>
+    /// <returns><see langword="true" /> if the file can be opened as <see cref="ZipArchive" />; otherwise, <see langword="false" />.</returns>
+    public static bool CanOpen(string filename) {
+      try {
+        using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read)) {
+          using (ZipArchive zip = new ZipArchive(fs)) {
+            return true;
+          }
+        }
+      } catch (InvalidDataException) {
+        return false;
+      }
+    }
   }
 }

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/MultiObjective/PearsonRSquaredAverageSimilarityEvaluator.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
+using HEAL.Attic;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -29 +28 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Parameters;
-using HEAL.Attic;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -36 +34 @@
   public class PearsonRSquaredAverageSimilarityEvaluator : SymbolicRegressionMultiObjectiveEvaluator {
     private const string StrictSimilarityParameterName = "StrictSimilarity";
+    private const string AverageSimilarityParameterName = "AverageSimilarity";
 
     private readonly object locker = new object();
 
+    private readonly SymbolicDataAnalysisExpressionTreeAverageSimilarityCalculator SimilarityCalculator = new SymbolicDataAnalysisExpressionTreeAverageSimilarityCalculator();
+
     public IFixedValueParameter<BoolValue> StrictSimilarityParameter {
       get { return (IFixedValueParameter<BoolValue>)Parameters[StrictSimilarityParameterName]; }
+    }
+
+    public ILookupParameter<DoubleValue> AverageSimilarityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[AverageSimilarityParameterName]; }
     }
 
@@ -58 +63 @@
     public PearsonRSquaredAverageSimilarityEvaluator() : base() {
       Parameters.Add(new FixedValueParameter<BoolValue>(StrictSimilarityParameterName, "Use strict similarity calculation.", new BoolValue(false)));
+      Parameters.Add(new LookupParameter<DoubleValue>(AverageSimilarityParameterName));
     }
 
-    public override IEnumerable<bool> Maximization { get { return new bool[2] { true, false }; } } // maximize R² and minimize model complexity 
+    public override IEnumerable<bool> Maximization { get { return new bool[2] { true, false }; } } // maximize R² and minimize average similarity 
 
     public override IOperation InstrumentedApply() {
@@ -73 +79 @@
         SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(interpreter, solution, problemData, rows, applyLinearScaling, ConstantOptimizationIterations, updateVariableWeights: ConstantOptimizationUpdateVariableWeights, lowerEstimationLimit: estimationLimits.Lower, upperEstimationLimit: estimationLimits.Upper);
       }
-      double[] qualities = Calculate(interpreter, solution, estimationLimits.Lower, estimationLimits.Upper, problemData, rows, applyLinearScaling, DecimalPlaces);
-      QualitiesParameter.ActualValue = new DoubleArray(qualities);
-      return base.InstrumentedApply();
-    }
 
-    public double[] Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree solution, double lowerEstimationLimit, double upperEstimationLimit, IRegressionProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling, int decimalPlaces) {
-      double r2 = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(interpreter, solution, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling);
-      if (decimalPlaces >= 0)
-        r2 = Math.Round(r2, decimalPlaces);
+      double r2 = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(interpreter, solution, estimationLimits.Lower, estimationLimits.Upper, problemData, rows, applyLinearScaling);
 
-      var variables = ExecutionContext.Scope.Variables;
-      if (!variables.ContainsKey("AverageSimilarity")) {
-        lock (locker) {
-          CalculateAverageSimilarities(ExecutionContext.Scope.Parent.SubScopes.Where(x => x.Variables.ContainsKey("SymbolicExpressionTree")).ToArray(), StrictSimilarity);
+      if (DecimalPlaces >= 0)
+        r2 = Math.Round(r2, DecimalPlaces);
 
+      lock (locker) {
+        if (AverageSimilarityParameter.ActualValue == null) {
+          var context = new ExecutionContext(null, SimilarityCalculator, ExecutionContext.Scope.Parent);
+          SimilarityCalculator.StrictSimilarity = StrictSimilarity;
+          SimilarityCalculator.Execute(context, CancellationToken);
         }
       }
+      var avgSimilarity = AverageSimilarityParameter.ActualValue.Value;
 
-      double avgSim = ((DoubleValue)variables["AverageSimilarity"].Value).Value;
-      return new double[2] { r2, avgSim };
+      QualitiesParameter.ActualValue = new DoubleArray(new[] { r2, avgSimilarity });
+      return base.InstrumentedApply();
     }
 
     public override double[] Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows) {
       SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context;
+      AverageSimilarityParameter.ExecutionContext = context;
       EstimationLimitsParameter.ExecutionContext = context;
       ApplyLinearScalingParameter.ExecutionContext = context;
-      // DecimalPlaces parameter is a FixedValueParameter and doesn't need the context.
 
-      double[] quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, problemData, rows, ApplyLinearScalingParameter.ActualValue.Value, DecimalPlaces);
+      var estimationLimits = EstimationLimitsParameter.ActualValue;
+      var applyLinearScaling = ApplyLinearScalingParameter.ActualValue.Value;
+
+      double r2 = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, estimationLimits.Lower, estimationLimits.Upper, problemData, rows, applyLinearScaling);
+
+      lock (locker) {
+        if (AverageSimilarityParameter.ActualValue == null) {
+          var ctx = new ExecutionContext(null, SimilarityCalculator, context.Scope.Parent);
+          SimilarityCalculator.StrictSimilarity = StrictSimilarity;
+          SimilarityCalculator.Execute(context, CancellationToken);
+        }
+      }
+      var avgSimilarity = AverageSimilarityParameter.ActualValue.Value;
 
       SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
@@ -107 +122 @@
       ApplyLinearScalingParameter.ExecutionContext = null;
 
-      return quality;
-    }
-
-    private readonly Stopwatch sw = new Stopwatch();
-    public void CalculateAverageSimilarities(IScope[] treeScopes, bool strict) {
-      var trees = treeScopes.Select(x => (ISymbolicExpressionTree)x.Variables["SymbolicExpressionTree"].Value).ToArray();
-      var similarityMatrix = SymbolicExpressionTreeHash.ComputeSimilarityMatrix(trees, simplify: false, strict: strict);
-
-      for (int i = 0; i < treeScopes.Length; ++i) {
-        var scope = treeScopes[i];
-        var avgSimilarity = 0d;
-        for (int j = 0; j < trees.Length; ++j) {
-          if (i == j) continue;
-          avgSimilarity += similarityMatrix[i, j];
-        }
-        avgSimilarity /= trees.Length - 1;
-
-        if (scope.Variables.ContainsKey("AverageSimilarity")) {
-          ((DoubleValue)scope.Variables["AverageSimilarity"].Value).Value = avgSimilarity;
-        } else {
-          scope.Variables.Add(new Core.Variable("AverageSimilarity", new DoubleValue(avgSimilarity)));
-        }
-      }
+      return new[] { r2, avgSimilarity };
     }
   }

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Crossovers/SymbolicDataAnalysisExpressionDiversityPreservingCrossover.cs

@@ -1 @@
-using System;
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-
+using HEAL.Attic;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -8 +29 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Parameters;
-using HEAL.Attic;
 using HeuristicLab.Random;
 using static HeuristicLab.Problems.DataAnalysis.Symbolic.SymbolicExpressionHashExtensions;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
-  [Item("DiversityCrossover", "Simple crossover operator prioritizing internal nodes according to the given probability.")]
+  [Item("DiversityCrossover", "Simple crossover operator preventing swap between subtrees with the same hash value.")]
   [StorableType("ED35B0D9-9704-4D32-B10B-8F9870E76781")]
   public sealed class SymbolicDataAnalysisExpressionDiversityPreservingCrossover<T> : SymbolicDataAnalysisExpressionCrossover<T> where T : class, IDataAnalysisProblemData {
@@ -20 +40 @@
     private const string WindowingParameterName = "Windowing";
     private const string ProportionalSamplingParameterName = "ProportionalSampling";
+    private const string StrictHashingParameterName = "StrictHashing";
 
     private static readonly Func<byte[], ulong> hashFunction = HashUtil.JSHash;
@@ -35 +56 @@
       get { return (IValueLookupParameter<BoolValue>)Parameters[ProportionalSamplingParameterName]; }
     }
+
+    public IFixedValueParameter<BoolValue> StrictHashingParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[StrictHashingParameterName]; }
+    }
     #endregion
 
@@ -49 +74 @@
       get { return ProportionalSamplingParameter.ActualValue; }
     }
+
+    bool StrictHashing {
+      get { return StrictHashingParameter.Value.Value; }
+    }
     #endregion
+
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(StrictHashingParameterName)) {
+        Parameters.Add(new FixedValueParameter<BoolValue>(StrictHashingParameterName, "Use strict hashing when calculating subtree hash values."));
+      }
+    }
 
     public SymbolicDataAnalysisExpressionDiversityPreservingCrossover() {
@@ -56 +93 @@
       Parameters.Add(new ValueLookupParameter<BoolValue>(WindowingParameterName, "Use proportional sampling with windowing for cutpoint selection.", new BoolValue(false)));
       Parameters.Add(new ValueLookupParameter<BoolValue>(ProportionalSamplingParameterName, "Select cutpoints proportionally using probabilities as weights instead of randomly.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(StrictHashingParameterName, "Use strict hashing when calculating subtree hash values."));
     }
 
@@ -72 +110 @@
     }
 
-    public static ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, double internalCrossoverPointProbability, int maxLength, int maxDepth, bool windowing, bool proportionalSampling = false) {
-      var leafCrossoverPointProbability = 1 - internalCrossoverPointProbability;
-
-      var nodes0 = ActualRoot(parent0).MakeNodes().Sort(hashFunction);
-      var nodes1 = ActualRoot(parent1).MakeNodes().Sort(hashFunction);
+    public static ISymbolicExpressionTree Cross(IRandom random, ISymbolicExpressionTree parent0, ISymbolicExpressionTree parent1, double internalCrossoverPointProbability, int maxLength, int maxDepth, bool windowing, bool proportionalSampling = false, bool strictHashing = false) {
+      var nodes0 = ActualRoot(parent0).MakeNodes(strictHashing).Sort(hashFunction);
+      var nodes1 = ActualRoot(parent1).MakeNodes(strictHashing).Sort(hashFunction);
 
       IList<HashNode<ISymbolicExpressionTreeNode>> sampled0;
@@ -126 +162 @@
       var proportionalSampling = ProportionalSampling.Value;
 
-      return Cross(random, parent0, parent1, internalCrossoverPointProbability, maxLength, maxDepth, windowing, proportionalSampling);
+      return Cross(random, parent0, parent1, internalCrossoverPointProbability, maxLength, maxDepth, windowing, proportionalSampling, StrictHashing);
     }
 

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashExtensions.cs

@@ -21 +21 @@
 
 using System;
-using System.Collections.Generic;
+using System.Linq;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
@@ -38 +38 @@
       public SimplifyAction Simplify;
 
-      public IComparer<T> Comparer;
+      //public IComparer<T> Comparer;
 
       public bool IsLeaf => Arity == 0;
 
-      public HashNode(IComparer<T> comparer) {
-        Comparer = comparer;
-      }
-
-      private HashNode() { }
+      //public HashNode(IComparer<T> comparer) {
+      //  Comparer = comparer;
+      //}
+
+      //public HashNode() { }
 
       public int CompareTo(HashNode<T> other) {
-        var res = Comparer.Compare(Data, other.Data);
-        return res == 0 ? CalculatedHashValue.CompareTo(other.CalculatedHashValue) : res;
+        return CalculatedHashValue.CompareTo(other.CalculatedHashValue);
       }
 
@@ -103 +102 @@
 
     public static HashNode<T>[] Simplify<T>(this HashNode<T>[] nodes, Func<byte[], ulong> hashFunction) where T : class {
-      var reduced = nodes.UpdateNodeSizes().Reduce().Sort(hashFunction);
-
-      for (int i = 0; i < reduced.Length; ++i) {
-        var node = reduced[i];
-        if (node.IsLeaf) {
-          continue;
-        }
-        node.Simplify?.Invoke(ref reduced, i);
-      }
-      // detect if anything was simplified
-      var count = 0;
-      foreach (var node in reduced) {
-        if (!node.Enabled) { ++count; }
-      }
-      if (count == 0) {
-        return reduced;
-      }
-
-      var simplified = new HashNode<T>[reduced.Length - count];
-      int j = 0;
-      foreach (var node in reduced) {
-        if (node.Enabled) {
-          simplified[j++] = node;
-        }
-      }
-      return simplified.UpdateNodeSizes().Reduce().Sort(hashFunction);
+      bool simplified = false;
+      nodes = nodes.UpdateNodeSizes().Reduce().Sort(hashFunction);
+      do {
+        if (simplified) {
+          simplified = false;
+          nodes = nodes.Where(x => x.Enabled).ToArray().UpdateNodeSizes().Reduce().Sort(hashFunction);
+        }
+
+        for (int i = 0; i < nodes.Length; ++i) {
+          var node = nodes[i];
+          if (node.IsLeaf) {
+            continue;
+          }
+          node.Simplify?.Invoke(ref nodes, i);
+          for (int j = i - node.Size; j < i; ++j) {
+            // detect if anything was simplified
+            if (!nodes[j].Enabled) {
+              simplified = true;
+              break;
+            }
+          }
+        }
+      } while (simplified);
+      return nodes.UpdateNodeSizes().Sort(hashFunction);
     }
 
@@ -207 +204 @@
     }
 
-    private static HashNode<T>[] Reduce<T>(this HashNode<T>[] nodes) where T : class {
+    public static HashNode<T>[] Reduce<T>(this HashNode<T>[] nodes) where T : class {
       int count = 0;
       for (int i = 0; i < nodes.Length; ++i) {

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/SymbolicExpressionTreeHash.cs

@@ -38 +38 @@
     private static readonly Constant constant = new Constant();
 
-    private static readonly ISymbolicExpressionTreeNodeComparer comparer = new SymbolicExpressionTreeNodeComparer();
     private static ISymbolicExpressionTreeNode ActualRoot(this ISymbolicExpressionTree tree) => tree.Root.GetSubtree(0).GetSubtree(0);
 
@@ -74 +73 @@
       }
       var hash = (ulong)name.GetHashCode();
-      var hashNode = new HashNode<ISymbolicExpressionTreeNode>(comparer) {
+      var hashNode = new HashNode<ISymbolicExpressionTreeNode> {
         Data = node,
         Arity = node.SubtreeCount,

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj

@@ -132 +132 @@
   <ItemGroup>
     <Compile Include="Analyzers\SymbolicDataAnalysisBottomUpDiversityAnalyzer.cs" />
-    <Compile Include="Analyzers\SymbolicDataAnalysisBuildingBlockAnalyzer.cs" />
     <Compile Include="Analyzers\SymbolicDataAnalysisSingleObjectivePruningAnalyzer.cs" />
     <Compile Include="Analyzers\SymbolicDataAnalysisSingleObjectiveValidationParetoBestSolutionAnalyzer.cs" />
@@ -173 +172 @@
     <Compile Include="Interpreter\SymbolicDataAnalysisExpressionTreeNativeInterpreter.cs" />
     <Compile Include="Selectors\DiversitySelector.cs" />
+    <Compile Include="SymbolicDataAnalysisExpressionTreeAverageSimilarityCalculator.cs" />
     <Compile Include="SymbolicDataAnalysisExpressionTreeSimplificationOperator.cs" />
     <Compile Include="SymbolicDataAnalysisModelComplexityCalculator.cs" />

branches/2994-AutoDiffForIntervals/HeuristicLab.Tests/HeuristicLab.Persistence.Attic/PersistenceConsistencyChecks.cs

@@ -18 +18 @@
       var dict = new Dictionary<Guid, string>();
       var duplicates = new Dictionary<string, string>();
-      //get all non-generic and instantiable classes which implement IContentView
       foreach (Type type in ApplicationManager.Manager.GetTypes(typeof(object))) {
         var attr = StorableTypeAttribute.GetStorableTypeAttribute(type);