Index: /branches/M5Regression/Build.cmd
===================================================================
--- /branches/M5Regression/Build.cmd (revision 15428)
+++ /branches/M5Regression/Build.cmd (revision 15428)
@@ -0,0 +1,104 @@
+@ECHO OFF
+
+SET CLEANBEFOREBUILD=
+
+SET SELECTED=
+SET CONFIGURATION=
+SET PLATFORM=
+
+IF "%~1"=="" GOTO :prompt_solution
+
+SET SELECTED=%1
+IF NOT EXIST %SELECTED% (
+ ECHO Solution file %SELECTED% could not be found.
+ GOTO :end
+)
+ECHO Building solution %SELECTED% ...
+GOTO :config_selection
+
+:prompt_solution
+SET /A COUNT=0
+FOR /F "tokens=*" %%A IN ('dir /B *.sln') DO (
+ CALL :forloopbody "%%A"
+)
+
+IF "%COUNT%"=="1" (
+ SET SELECTED=%SOLUTIONS.1%
+ ECHO Building %SOLUTIONS.1% as it is the only solution that was found ...
+ GOTO :config_selection
+)
+
+ECHO Found the following solutions:
+FOR /F "tokens=2* delims=.=" %%A IN ('SET SOLUTIONS.') DO ECHO %%A = %%B
+ECHO.
+SET /P SOLUTIONINDEX=Which solution to build? Type the number:
+
+SET SELECTED=""
+FOR /F "tokens=2* delims=.=" %%A IN ('SET SOLUTIONS.') DO (
+ IF "%%A"=="%SOLUTIONINDEX%" SET SELECTED=%%B
+)
+
+IF %SELECTED%=="" GOTO :eof
+
+:config_selection
+IF "%~2"=="" GOTO :prompt_config
+
+SET CONFIGURATION=%~2
+ECHO Building configuration %CONFIGURATION% ...
+GOTO :platform_selection
+
+:prompt_config
+SET /P CONFIGURATION=Which configuration to build [Release]:
+IF "%CONFIGURATION%"=="" SET CONFIGURATION=Release
+
+:platform_selection
+IF "%~3"=="" GOTO :prompt_platform
+
+SET PLATFORM=%~3
+ECHO Building platform %PLATFORM% ...
+GOTO :clean
+
+:prompt_platform
+SET /P PLATFORM=Which platform to build [Any CPU]:
+IF "%PLATFORM%"=="" SET PLATFORM=Any CPU
+
+:clean
+IF "%~4"=="" GOTO :prompt_clean
+
+SET CLEANBEFOREBUILD=%~4
+GOTO :main
+
+:prompt_clean
+SET /P CLEANBEFOREBUILD=Would you like to clean before building [n]:
+IF "%CLEANBEFOREBUILD%"=="" SET CLEANBEFOREBUILD=n
+
+:main
+REM First find the path to the msbuild.exe by performing a registry query
+FOR /F "tokens=1,3 delims= " %%A IN ('REG QUERY "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\4.0"') DO (
+ IF "%%A"=="MSBuildToolsPath" SET MSBUILDPATH=%%B)
+
+REM Then execute msbuild to clean and build the solution
+REM Disable that msbuild creates a cache file of the solution
+SET MSBuildUseNoSolutionCache=1
+REM Run msbuild to clean and then build
+IF "%CLEANBEFOREBUILD%" NEQ "n" (
+ ECHO Cleaning ...
+ %MSBUILDPATH%msbuild.exe %SELECTED% /target:Clean /p:Configuration="%CONFIGURATION%",Platform="%PLATFORM%" /m:2 /nologo /verbosity:q /clp:ErrorsOnly
+)
+ECHO Building ...
+%MSBUILDPATH%msbuild.exe %SELECTED% /target:Build /p:Configuration="%CONFIGURATION%",Platform="%PLATFORM%" /m:2 /nologo /verbosity:q /clp:ErrorsOnly
+
+ECHO.
+ECHO DONE.
+
+:end
+
+PAUSE
+
+GOTO :eof
+
+REM This workaround is necessary so that COUNT gets reevaluated
+:forloopbody
+SET /A COUNT+=1
+SET SOLUTIONS.%COUNT%=%1
+GOTO :eof
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationModel.cs (revision 15428)
@@ -0,0 +1,103 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item("OneFactor Classification Model", "A model that uses only one categorical feature (factor) to determine the class.")]
+ public sealed class OneFactorClassificationModel : ClassificationModel {
+ public override IEnumerable VariablesUsedForPrediction {
+ get { return new[] { Variable }; }
+ }
+
+ [Storable]
+ private string variable;
+ public string Variable {
+ get { return variable; }
+ }
+
+ [Storable]
+ private string[] variableValues;
+ public string[] VariableValues {
+ get { return variableValues; }
+ }
+
+ [Storable]
+ private double[] classes;
+ public double[] Classes {
+ get { return classes; }
+ }
+
+ [Storable]
+ private double defaultClass;
+ public double DefaultClass {
+ get { return defaultClass; }
+ }
+
+ [StorableConstructor]
+ private OneFactorClassificationModel(bool deserializing) : base(deserializing) { }
+ private OneFactorClassificationModel(OneFactorClassificationModel original, Cloner cloner)
+ : base(original, cloner) {
+ this.variable = (string)original.variable;
+ this.variableValues = (string[])original.variableValues.Clone();
+ this.classes = (double[])original.classes.Clone();
+ this.defaultClass = original.defaultClass;
+ }
+ public override IDeepCloneable Clone(Cloner cloner) { return new OneFactorClassificationModel(this, cloner); }
+
+ public OneFactorClassificationModel(string targetVariable, string variable, string[] variableValues, double[] classes, double defaultClass = double.NaN)
+ : base(targetVariable) {
+ if (variableValues.Length != classes.Length) {
+ throw new ArgumentException("Number of variable values and classes has to be equal.");
+ }
+ this.name = ItemName;
+ this.description = ItemDescription;
+ this.variable = variable;
+ this.variableValues = variableValues;
+ this.classes = classes;
+ this.defaultClass = double.IsNaN(defaultClass) ? classes.First() : defaultClass;
+ Array.Sort(variableValues, classes);
+ }
+
+ public override IEnumerable GetEstimatedClassValues(IDataset dataset, IEnumerable rows) {
+ return dataset.GetStringValues(Variable, rows)
+ .Select(GetPredictedValueForInput);
+ }
+
+ private double GetPredictedValueForInput(string val) {
+ var matchingIdx = Array.BinarySearch(variableValues, val);
+ if (matchingIdx >= 0) return classes[matchingIdx];
+ else return DefaultClass;
+ }
+
+ public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+ return new OneFactorClassificationSolution(this, new ClassificationProblemData(problemData));
+ }
+
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneFactorClassificationSolution.cs (revision 15428)
@@ -0,0 +1,48 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single factor for class prediction.")]
+ public sealed class OneFactorClassificationSolution : ClassificationSolution {
+ public new OneFactorClassificationModel Model {
+ get { return (OneFactorClassificationModel)base.Model; }
+ set { base.Model = value; }
+ }
+
+ [StorableConstructor]
+ private OneFactorClassificationSolution(bool deserializing) : base(deserializing) { }
+ private OneFactorClassificationSolution(OneFactorClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+ public OneFactorClassificationSolution(OneFactorClassificationModel model, IClassificationProblemData problemData)
+ : base(model, problemData) {
+ RecalculateResults();
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new OneFactorClassificationSolution(this, cloner);
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneR.cs (revision 15428)
@@ -0,0 +1,238 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ ///
+ /// 1R classification algorithm.
+ ///
+ [Item("OneR Classification", "A simple classification algorithm the searches the best single-variable split (does not support categorical features correctly). See R.C. Holte (1993). Very simple classification rules perform well on most commonly used datasets. Machine Learning. 11:63-91.")]
+ [StorableClass]
+ public sealed class OneR : FixedDataAnalysisAlgorithm {
+
+ public IValueParameter MinBucketSizeParameter {
+ get { return (IValueParameter)Parameters["MinBucketSize"]; }
+ }
+
+ [StorableConstructor]
+ private OneR(bool deserializing) : base(deserializing) { }
+
+ private OneR(OneR original, Cloner cloner)
+ : base(original, cloner) { }
+
+ public OneR()
+ : base() {
+ Parameters.Add(new ValueParameter("MinBucketSize", "Minimum size of a bucket for numerical values. (Except for the rightmost bucket)", new IntValue(6)));
+ Problem = new ClassificationProblem();
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new OneR(this, cloner);
+ }
+
+ protected override void Run(CancellationToken cancellationToken) {
+ var solution = CreateOneRSolution(Problem.ProblemData, MinBucketSizeParameter.Value.Value);
+ Results.Add(new Result("OneR solution", "The 1R classifier.", solution));
+ }
+
+ public static IClassificationSolution CreateOneRSolution(IClassificationProblemData problemData, int minBucketSize = 6) {
+ var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+ var model1 = FindBestDoubleVariableModel(problemData, minBucketSize);
+ var model2 = FindBestFactorModel(problemData);
+
+ if (model1 == null && model2 == null) throw new InvalidProgramException("Could not create OneR solution");
+ else if (model1 == null) return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+ else if (model2 == null) return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+ else {
+ var model1EstimatedValues = model1.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+ var model1NumCorrect = classValues.Zip(model1EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+ var model2EstimatedValues = model2.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+ var model2NumCorrect = classValues.Zip(model2EstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+
+ if (model1NumCorrect > model2NumCorrect) {
+ return new OneRClassificationSolution(model1, (IClassificationProblemData)problemData.Clone());
+ } else {
+ return new OneFactorClassificationSolution(model2, (IClassificationProblemData)problemData.Clone());
+ }
+ }
+ }
+
+ private static OneRClassificationModel FindBestDoubleVariableModel(IClassificationProblemData problemData, int minBucketSize = 6) {
+ var bestClassified = 0;
+ List bestSplits = null;
+ string bestVariable = string.Empty;
+ double bestMissingValuesClass = double.NaN;
+ var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+
+ var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType);
+
+ if (!allowedInputVariables.Any()) return null;
+
+ foreach (var variable in allowedInputVariables) {
+ var inputValues = problemData.Dataset.GetDoubleValues(variable, problemData.TrainingIndices);
+ var samples = inputValues.Zip(classValues, (i, v) => new Sample(i, v)).OrderBy(s => s.inputValue);
+
+ var missingValuesDistribution = samples
+ .Where(s => double.IsNaN(s.inputValue)).GroupBy(s => s.classValue)
+ .ToDictionary(s => s.Key, s => s.Count())
+ .MaxItems(s => s.Value)
+ .FirstOrDefault();
+
+ //calculate class distributions for all distinct inputValues
+ List> classDistributions = new List>();
+ List thresholds = new List();
+ double lastValue = double.NaN;
+ foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+ if (sample.inputValue > lastValue || double.IsNaN(lastValue)) {
+ if (!double.IsNaN(lastValue)) thresholds.Add((lastValue + sample.inputValue) / 2);
+ lastValue = sample.inputValue;
+ classDistributions.Add(new Dictionary());
+ foreach (var classValue in problemData.ClassValues)
+ classDistributions[classDistributions.Count - 1][classValue] = 0;
+
+ }
+ classDistributions[classDistributions.Count - 1][sample.classValue]++;
+ }
+ thresholds.Add(double.PositiveInfinity);
+
+ var distribution = classDistributions[0];
+ var threshold = thresholds[0];
+ var splits = new List();
+
+ for (int i = 1; i < classDistributions.Count; i++) {
+ var samplesInSplit = distribution.Max(d => d.Value);
+ //join splits if there are too few samples in the split or the distributions has the same maximum class value as the current split
+ if (samplesInSplit < minBucketSize ||
+ classDistributions[i].MaxItems(d => d.Value).Select(d => d.Key).Contains(
+ distribution.MaxItems(d => d.Value).Select(d => d.Key).First())) {
+ foreach (var classValue in classDistributions[i])
+ distribution[classValue.Key] += classValue.Value;
+ threshold = thresholds[i];
+ } else {
+ splits.Add(new Split(threshold, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+ distribution = classDistributions[i];
+ threshold = thresholds[i];
+ }
+ }
+ splits.Add(new Split(double.PositiveInfinity, distribution.MaxItems(d => d.Value).Select(d => d.Key).First()));
+
+ int correctClassified = 0;
+ int splitIndex = 0;
+ foreach (var sample in samples.Where(s => !double.IsNaN(s.inputValue))) {
+ while (sample.inputValue >= splits[splitIndex].thresholdValue)
+ splitIndex++;
+ correctClassified += sample.classValue.IsAlmost(splits[splitIndex].classValue) ? 1 : 0;
+ }
+ correctClassified += missingValuesDistribution.Value;
+
+ if (correctClassified > bestClassified) {
+ bestClassified = correctClassified;
+ bestSplits = splits;
+ bestVariable = variable;
+ bestMissingValuesClass = missingValuesDistribution.Value == 0 ? double.NaN : missingValuesDistribution.Key;
+ }
+ }
+
+ //remove neighboring splits with the same class value
+ for (int i = 0; i < bestSplits.Count - 1; i++) {
+ if (bestSplits[i].classValue.IsAlmost(bestSplits[i + 1].classValue)) {
+ bestSplits.Remove(bestSplits[i]);
+ i--;
+ }
+ }
+
+ var model = new OneRClassificationModel(problemData.TargetVariable, bestVariable,
+ bestSplits.Select(s => s.thresholdValue).ToArray(),
+ bestSplits.Select(s => s.classValue).ToArray(), bestMissingValuesClass);
+
+ return model;
+ }
+ private static OneFactorClassificationModel FindBestFactorModel(IClassificationProblemData problemData) {
+ var classValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+ var defaultClass = FindMostFrequentClassValue(classValues);
+ // only select string variables
+ var allowedInputVariables = problemData.AllowedInputVariables.Where(problemData.Dataset.VariableHasType);
+
+ if (!allowedInputVariables.Any()) return null;
+
+ OneFactorClassificationModel bestModel = null;
+ var bestModelNumCorrect = 0;
+
+ foreach (var variable in allowedInputVariables) {
+ var variableValues = problemData.Dataset.GetStringValues(variable, problemData.TrainingIndices);
+ var groupedClassValues = variableValues
+ .Zip(classValues, (v, c) => new KeyValuePair(v, c))
+ .GroupBy(kvp => kvp.Key)
+ .ToDictionary(g => g.Key, g => FindMostFrequentClassValue(g.Select(kvp => kvp.Value)));
+
+ var model = new OneFactorClassificationModel(problemData.TargetVariable, variable,
+ groupedClassValues.Select(kvp => kvp.Key).ToArray(), groupedClassValues.Select(kvp => kvp.Value).ToArray(), defaultClass);
+
+ var modelEstimatedValues = model.GetEstimatedClassValues(problemData.Dataset, problemData.TrainingIndices);
+ var modelNumCorrect = classValues.Zip(modelEstimatedValues, (a, b) => a.IsAlmost(b)).Count(e => e);
+ if (modelNumCorrect > bestModelNumCorrect) {
+ bestModelNumCorrect = modelNumCorrect;
+ bestModel = model;
+ }
+ }
+
+ return bestModel;
+ }
+
+ private static double FindMostFrequentClassValue(IEnumerable classValues) {
+ return classValues.GroupBy(c => c).OrderByDescending(g => g.Count()).Select(g => g.Key).First();
+ }
+
+ #region helper classes
+ private class Split {
+ public double thresholdValue;
+ public double classValue;
+
+ public Split(double thresholdValue, double classValue) {
+ this.thresholdValue = thresholdValue;
+ this.classValue = classValue;
+ }
+ }
+
+ private class Sample {
+ public double inputValue;
+ public double classValue;
+
+ public Sample(double inputValue, double classValue) {
+ this.inputValue = inputValue;
+ this.classValue = classValue;
+ }
+ }
+ #endregion
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationModel.cs (revision 15428)
@@ -0,0 +1,120 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item("OneR Classification Model", "A model that uses intervals for one variable to determine the class.")]
+ public sealed class OneRClassificationModel : ClassificationModel {
+ public override IEnumerable VariablesUsedForPrediction {
+ get { return new[] { Variable }; }
+ }
+
+ [Storable]
+ private string variable;
+ public string Variable {
+ get { return variable; }
+ }
+
+ [Storable]
+ private double[] splits;
+ public double[] Splits {
+ get { return splits; }
+ }
+
+ [Storable]
+ private double[] classes;
+ public double[] Classes {
+ get { return classes; }
+ }
+
+ [Storable]
+ private double missingValuesClass;
+ public double MissingValuesClass {
+ get { return missingValuesClass; }
+ }
+
+ [StorableConstructor]
+ private OneRClassificationModel(bool deserializing) : base(deserializing) { }
+ private OneRClassificationModel(OneRClassificationModel original, Cloner cloner)
+ : base(original, cloner) {
+ this.variable = (string)original.variable;
+ this.splits = (double[])original.splits.Clone();
+ this.classes = (double[])original.classes.Clone();
+ this.missingValuesClass = original.missingValuesClass;
+ }
+ public override IDeepCloneable Clone(Cloner cloner) { return new OneRClassificationModel(this, cloner); }
+
+ public OneRClassificationModel(string targetVariable, string variable, double[] splits, double[] classes, double missingValuesClass = double.NaN)
+ : base(targetVariable) {
+ if (splits.Length != classes.Length) {
+ throw new ArgumentException("Number of splits and classes has to be equal.");
+ }
+ if (!Double.IsPositiveInfinity(splits[splits.Length - 1])) {
+ throw new ArgumentException("Last split has to be double.PositiveInfinity, so that all values are covered.");
+ }
+ this.name = ItemName;
+ this.description = ItemDescription;
+ this.variable = variable;
+ this.splits = splits;
+ this.classes = classes;
+ this.missingValuesClass = missingValuesClass;
+ }
+
+ // uses sorting to return the values in the order of rows, instead of using nested for loops
+ // to avoid O(n²) runtime
+ public override IEnumerable GetEstimatedClassValues(IDataset dataset, IEnumerable rows) {
+ var values = dataset.GetDoubleValues(Variable, rows).ToArray();
+ var rowsArray = rows.ToArray();
+ var order = Enumerable.Range(0, rowsArray.Length).ToArray();
+ double[] estimated = new double[rowsArray.Length];
+ Array.Sort(rowsArray, order);
+ Array.Sort(values, rowsArray);
+ int curSplit = 0, curIndex = 0;
+ while (curIndex < values.Length && Double.IsNaN(values[curIndex])) {
+ estimated[curIndex] = MissingValuesClass;
+ curIndex++;
+ }
+ while (curSplit < Splits.Length) {
+ while (curIndex < values.Length && Splits[curSplit] > values[curIndex]) {
+ estimated[curIndex] = classes[curSplit];
+ curIndex++;
+ }
+ curSplit++;
+ }
+ Array.Sort(rowsArray, estimated);
+ Array.Sort(order, estimated);
+ return estimated;
+ }
+
+ public override IClassificationSolution CreateClassificationSolution(IClassificationProblemData problemData) {
+ return new OneRClassificationSolution(this, new ClassificationProblemData(problemData));
+ }
+
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/OneRClassificationSolution.cs (revision 15428)
@@ -0,0 +1,48 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "OneR Classification Solution", Description = "Represents a OneR classification solution which uses only a single feature with potentially multiple thresholds for class prediction.")]
+ public sealed class OneRClassificationSolution : ClassificationSolution {
+ public new OneRClassificationModel Model {
+ get { return (OneRClassificationModel)base.Model; }
+ set { base.Model = value; }
+ }
+
+ [StorableConstructor]
+ private OneRClassificationSolution(bool deserializing) : base(deserializing) { }
+ private OneRClassificationSolution(OneRClassificationSolution original, Cloner cloner) : base(original, cloner) { }
+ public OneRClassificationSolution(OneRClassificationModel model, IClassificationProblemData problemData)
+ : base(model, problemData) {
+ RecalculateResults();
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new OneRClassificationSolution(this, cloner);
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/BaselineClassifiers/ZeroR.cs (revision 15428)
@@ -0,0 +1,72 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ ///
+ /// 0R classification algorithm.
+ ///
+ [Item("ZeroR Classification", "The simplest possible classifier, ZeroR always predicts the majority class.")]
+ [StorableClass]
+ public sealed class ZeroR : FixedDataAnalysisAlgorithm {
+
+ [StorableConstructor]
+ private ZeroR(bool deserializing) : base(deserializing) { }
+ private ZeroR(ZeroR original, Cloner cloner)
+ : base(original, cloner) {
+ }
+ public ZeroR()
+ : base() {
+ Problem = new ClassificationProblem();
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new ZeroR(this, cloner);
+ }
+
+ protected override void Run(CancellationToken cancellationToken) {
+ var solution = CreateZeroRSolution(Problem.ProblemData);
+ Results.Add(new Result("ZeroR solution", "The simplest possible classifier, ZeroR always predicts the majority class.", solution));
+ }
+
+ public static IClassificationSolution CreateZeroRSolution(IClassificationProblemData problemData) {
+ var dataset = problemData.Dataset;
+ string target = problemData.TargetVariable;
+ var targetValues = dataset.GetDoubleValues(target, problemData.TrainingIndices);
+
+
+ // if multiple classes have the same number of observations then simply take the first one
+ var dominantClass = targetValues.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count())
+ .MaxItems(kvp => kvp.Value).Select(x => x.Key).First();
+
+ var model = new ConstantModel(dominantClass, target);
+ var solution = model.CreateClassificationSolution(problemData);
+ return solution;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs (revision 15428)
@@ -0,0 +1,813 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using HeuristicLab.Collections;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [Item("Cross Validation (CV)", "Cross-validation wrapper for data analysis algorithms.")]
+ [Creatable(CreatableAttribute.Categories.DataAnalysis, Priority = 100)]
+ [StorableClass]
+ public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
+ [Storable]
+ private int seed;
+
+ private SemaphoreSlim availableWorkers; // limits the number of concurrent algorithm executions
+ private ManualResetEventSlim allAlgorithmsFinished; // this indicates that all started algorithms have been paused or stopped
+
+ public CrossValidation()
+ : base() {
+ name = ItemName;
+ description = ItemDescription;
+
+ executionState = ExecutionState.Stopped;
+ runs = new RunCollection { OptimizerName = name };
+ runsCounter = 0;
+
+ algorithm = null;
+ clonedAlgorithms = new ItemCollection();
+ results = new ResultCollection();
+
+ folds = new IntValue(2);
+ numberOfWorkers = new IntValue(1);
+ samplesStart = new IntValue(0);
+ samplesEnd = new IntValue(0);
+ shuffleSamples = new BoolValue(false);
+ storeAlgorithmInEachRun = false;
+
+ RegisterEvents();
+ if (Algorithm != null) RegisterAlgorithmEvents();
+ }
+
+ public string Filename { get; set; }
+
+ #region persistence and cloning
+ [StorableConstructor]
+ private CrossValidation(bool deserializing)
+ : base(deserializing) {
+ }
+ [StorableHook(HookType.AfterDeserialization)]
+ private void AfterDeserialization() {
+ // BackwardsCompatibility3.3
+ #region Backwards compatible code, remove with 3.4
+ if (shuffleSamples == null) shuffleSamples = new BoolValue(false);
+ #endregion
+
+ RegisterEvents();
+ if (Algorithm != null) RegisterAlgorithmEvents();
+ }
+
+ private CrossValidation(CrossValidation original, Cloner cloner)
+ : base(original, cloner) {
+ executionState = original.executionState;
+ storeAlgorithmInEachRun = original.storeAlgorithmInEachRun;
+ runs = cloner.Clone(original.runs);
+ runsCounter = original.runsCounter;
+ algorithm = cloner.Clone(original.algorithm);
+ clonedAlgorithms = cloner.Clone(original.clonedAlgorithms);
+ results = cloner.Clone(original.results);
+
+ folds = cloner.Clone(original.folds);
+ numberOfWorkers = cloner.Clone(original.numberOfWorkers);
+ samplesStart = cloner.Clone(original.samplesStart);
+ samplesEnd = cloner.Clone(original.samplesEnd);
+ shuffleSamples = cloner.Clone(original.shuffleSamples);
+ seed = original.seed;
+
+ RegisterEvents();
+ if (Algorithm != null) RegisterAlgorithmEvents();
+ }
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CrossValidation(this, cloner);
+ }
+
+ #endregion
+
+ #region properties
+ [Storable]
+ private IAlgorithm algorithm;
+ public IAlgorithm Algorithm {
+ get { return algorithm; }
+ set {
+ if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+ throw new InvalidOperationException("Changing the algorithm is only allowed if the CrossValidation is stopped or prepared.");
+ if (algorithm != value) {
+ if (value != null && value.Problem != null && !(value.Problem is IDataAnalysisProblem))
+ throw new ArgumentException("Only algorithms with a DataAnalysisProblem could be used for the cross validation.");
+ if (algorithm != null) DeregisterAlgorithmEvents();
+ algorithm = value;
+ Parameters.Clear();
+
+ if (algorithm != null) {
+ algorithm.StoreAlgorithmInEachRun = false;
+ RegisterAlgorithmEvents();
+ algorithm.Prepare(true);
+ Parameters.AddRange(algorithm.Parameters);
+ }
+ OnAlgorithmChanged();
+ Prepare();
+ }
+ }
+ }
+
+
+ [Storable]
+ private IDataAnalysisProblem problem;
+ public IDataAnalysisProblem Problem {
+ get {
+ if (algorithm == null)
+ return null;
+ return (IDataAnalysisProblem)algorithm.Problem;
+ }
+ set {
+ if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped)
+ throw new InvalidOperationException("Changing the problem is only allowed if the CrossValidation is stopped or prepared.");
+ if (algorithm == null) throw new ArgumentNullException("Could not set a problem before an algorithm was set.");
+ algorithm.Problem = value;
+ problem = value;
+ }
+ }
+
+ IProblem IAlgorithm.Problem {
+ get { return Problem; }
+ set {
+ if (value != null && !ProblemType.IsInstanceOfType(value))
+ throw new ArgumentException("Only DataAnalysisProblems could be used for the cross validation.");
+ Problem = (IDataAnalysisProblem)value;
+ }
+ }
+ public Type ProblemType {
+ get { return typeof(IDataAnalysisProblem); }
+ }
+
+ [Storable]
+ private ItemCollection clonedAlgorithms;
+
+ public IEnumerable NestedOptimizers {
+ get {
+ if (Algorithm == null) yield break;
+ yield return Algorithm;
+ }
+ }
+
+ [Storable]
+ private ResultCollection results;
+ public ResultCollection Results {
+ get { return results; }
+ }
+ [Storable]
+ private BoolValue shuffleSamples;
+ public BoolValue ShuffleSamples {
+ get { return shuffleSamples; }
+ }
+ [Storable]
+ private IntValue folds;
+ public IntValue Folds {
+ get { return folds; }
+ }
+ [Storable]
+ private IntValue samplesStart;
+ public IntValue SamplesStart {
+ get { return samplesStart; }
+ }
+ [Storable]
+ private IntValue samplesEnd;
+ public IntValue SamplesEnd {
+ get { return samplesEnd; }
+ }
+ [Storable]
+ private IntValue numberOfWorkers;
+ public IntValue NumberOfWorkers {
+ get { return numberOfWorkers; }
+ }
+
+ [Storable]
+ private bool storeAlgorithmInEachRun;
+ public bool StoreAlgorithmInEachRun {
+ get { return storeAlgorithmInEachRun; }
+ set {
+ if (storeAlgorithmInEachRun != value) {
+ storeAlgorithmInEachRun = value;
+ OnStoreAlgorithmInEachRunChanged();
+ }
+ }
+ }
+
+ [Storable]
+ private int runsCounter;
+ [Storable]
+ private RunCollection runs;
+ public RunCollection Runs {
+ get { return runs; }
+ }
+
+ [Storable]
+ private ExecutionState executionState;
+ public ExecutionState ExecutionState {
+ get { return executionState; }
+ private set {
+ if (executionState != value) {
+ executionState = value;
+ OnExecutionStateChanged();
+ OnItemImageChanged();
+ }
+ }
+ }
+ public static new Image StaticItemImage {
+ get { return HeuristicLab.Common.Resources.VSImageLibrary.Event; }
+ }
+ public override Image ItemImage {
+ get {
+ if (ExecutionState == ExecutionState.Prepared) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePrepared;
+ else if (ExecutionState == ExecutionState.Started) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStarted;
+ else if (ExecutionState == ExecutionState.Paused) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutablePaused;
+ else if (ExecutionState == ExecutionState.Stopped) return HeuristicLab.Common.Resources.VSImageLibrary.ExecutableStopped;
+ else return base.ItemImage;
+ }
+ }
+
+ public TimeSpan ExecutionTime {
+ get {
+ if (ExecutionState != ExecutionState.Prepared)
+ return TimeSpan.FromMilliseconds(clonedAlgorithms.Select(x => x.ExecutionTime.TotalMilliseconds).Sum());
+ return TimeSpan.Zero;
+ }
+ }
+ #endregion
+
+ protected override void OnNameChanged() {
+ base.OnNameChanged();
+ Runs.OptimizerName = Name;
+ }
+
+ public void Prepare() {
+ if (startPending) return;
+ if (ExecutionState == ExecutionState.Started)
+ throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState));
+ results.Clear();
+ clonedAlgorithms.Clear();
+ if (Algorithm != null) {
+ Algorithm.Prepare();
+ if (Algorithm.ExecutionState == ExecutionState.Prepared) OnPrepared();
+ }
+ }
+ public void Prepare(bool clearRuns) {
+ if (clearRuns) runs.Clear();
+ Prepare();
+ }
+
+ private bool startPending;
+ public void Start() {
+ Start(CancellationToken.None);
+ }
+ public void Start(CancellationToken cancellationToken) {
+ lock (locker) {
+ if (startPending) return;
+ startPending = true;
+ }
+
+ try {
+ if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused))
+ throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
+ seed = new FastRandom().NextInt();
+
+ if (Algorithm == null) return;
+ //create cloned algorithms
+ if (clonedAlgorithms.Count == 0) {
+ int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
+ IDataset shuffledDataset = null;
+ for (int i = 0; i < Folds.Value; i++) {
+ var cloner = new Cloner();
+ if (ShuffleSamples.Value) {
+ var random = new FastRandom(seed);
+ var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
+ var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
+ shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
+ cloner.RegisterClonedObject(dataset, shuffledDataset);
+ }
+ IAlgorithm clonedAlgorithm = cloner.Clone(Algorithm);
+ clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
+ IDataAnalysisProblem problem = clonedAlgorithm.Problem as IDataAnalysisProblem;
+ ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
+
+ int testStart = (i * testSamplesCount) + SamplesStart.Value;
+ int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
+
+ problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
+ problem.ProblemData.TrainingPartition.End = SamplesEnd.Value;
+ problem.ProblemData.TestPartition.Start = testStart;
+ problem.ProblemData.TestPartition.End = testEnd;
+ DataAnalysisProblemData problemData = problem.ProblemData as DataAnalysisProblemData;
+ if (problemData != null) {
+ problemData.TrainingPartitionParameter.Hidden = false;
+ problemData.TestPartitionParameter.Hidden = false;
+ }
+
+ if (symbolicProblem != null) {
+ symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+ symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+ }
+ clonedAlgorithm.Prepare();
+ clonedAlgorithms.Add(clonedAlgorithm);
+ }
+ }
+
+ OnStarted();
+ } finally {
+ if (startPending) startPending = false;
+ }
+
+ availableWorkers = new SemaphoreSlim(NumberOfWorkers.Value, NumberOfWorkers.Value);
+ allAlgorithmsFinished = new ManualResetEventSlim(false);
+
+ var startedTasks = new List(clonedAlgorithms.Count);
+
+ //start prepared or paused cloned algorithms
+ foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
+ if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
+ if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
+ clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
+ availableWorkers.Wait();
+ lock (locker) {
+ if (pausePending || stopPending || ExecutionState != ExecutionState.Started) break;
+ var task = clonedAlgorithm.StartAsync(cancellationToken);
+ startedTasks.Add(task);
+ }
+ }
+ }
+
+ allAlgorithmsFinished.Wait();
+
+ Task.WaitAll(startedTasks.ToArray()); // to get exceptions not handled within the tasks
+ }
+
+ public async Task StartAsync() { await StartAsync(CancellationToken.None); }
+ public async Task StartAsync(CancellationToken cancellationToken) {
+ await AsyncHelper.DoAsync(Start, cancellationToken);
+ }
+
+ private bool pausePending;
+ public void Pause() {
+ if (startPending) return;
+ if (ExecutionState != ExecutionState.Started)
+ throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState));
+ if (!pausePending) {
+ pausePending = true;
+ lock (locker) {
+ var toPause = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started).ToList();
+ foreach (var optimizer in toPause) {
+ // a race-condition may occur when the optimizer has changed the state by itself in the meantime
+ try { optimizer.Pause(); } catch (InvalidOperationException) { }
+ }
+ }
+ }
+ }
+
+ private bool stopPending;
+ public void Stop() {
+ if (startPending) return;
+ if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused))
+ throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".",
+ ExecutionState));
+ if (!stopPending) {
+ stopPending = true;
+ lock (locker) {
+ var toStop = clonedAlgorithms.Where(x => x.ExecutionState == ExecutionState.Started || x.ExecutionState == ExecutionState.Paused).ToList();
+ foreach (var optimizer in toStop) {
+ // a race-condition may occur when the optimizer has changed the state by itself in the meantime
+ try { optimizer.Stop(); } catch (InvalidOperationException) { }
+ }
+ }
+ }
+ }
+
+ #region collect parameters and results
+ public override void CollectParameterValues(IDictionary values) {
+ values.Add("Algorithm Name", new StringValue(Name));
+ values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
+ values.Add("Folds", new IntValue(Folds.Value));
+
+ if (algorithm != null) {
+ values.Add("CrossValidation Algorithm Name", new StringValue(Algorithm.Name));
+ values.Add("CrossValidation Algorithm Type", new StringValue(Algorithm.GetType().GetPrettyName()));
+ base.CollectParameterValues(values);
+ }
+ if (Problem != null) {
+ values.Add("Problem Name", new StringValue(Problem.Name));
+ values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName()));
+ Problem.CollectParameterValues(values);
+ }
+ }
+
+ public void CollectResultValues(IDictionary results) {
+ var clonedResults = (ResultCollection)this.results.Clone();
+ foreach (var result in clonedResults) {
+ results.Add(result.Name, result.Value);
+ }
+ }
+
+ private void AggregateResultValues(IDictionary results) {
+ IEnumerable runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
+ IEnumerable> resultCollections = runs.Where(x => x != null).SelectMany(x => x.Results).ToList();
+
+ foreach (IResult result in ExtractAndAggregateResults(resultCollections))
+ results.Add(result.Name, result.Value);
+ foreach (IResult result in ExtractAndAggregateResults(resultCollections))
+ results.Add(result.Name, result.Value);
+ foreach (IResult result in ExtractAndAggregateResults(resultCollections))
+ results.Add(result.Name, result.Value);
+ foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
+ results.Add(result.Name, result.Value);
+ }
+ foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
+ results.Add(result.Name, result.Value);
+ }
+ results.Add("Execution Time", new TimeSpanValue(this.ExecutionTime));
+ results.Add("CrossValidation Folds", new RunCollection(runs));
+ }
+
+ private IEnumerable ExtractAndAggregateRegressionSolutions(IEnumerable> resultCollections) {
+ Dictionary> resultSolutions = new Dictionary>();
+ foreach (var result in resultCollections) {
+ var regressionSolution = result.Value as IRegressionSolution;
+ if (regressionSolution != null) {
+ if (resultSolutions.ContainsKey(result.Key)) {
+ resultSolutions[result.Key].Add(regressionSolution);
+ } else {
+ resultSolutions.Add(result.Key, new List() { regressionSolution });
+ }
+ }
+ }
+ List aggregatedResults = new List();
+ foreach (KeyValuePair> solutions in resultSolutions) {
+ // clone manually to correctly clone references between cloned root objects
+ Cloner cloner = new Cloner();
+ if (ShuffleSamples.Value) {
+ var dataset = (Dataset)Problem.ProblemData.Dataset;
+ var random = new FastRandom(seed);
+ var shuffledDataset = dataset.Shuffle(random);
+ cloner.RegisterClonedObject(dataset, shuffledDataset);
+ }
+ var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
+ // set partitions of problem data clone correctly
+ problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+ problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+ // clone models
+ var ensembleSolution = new RegressionEnsembleSolution(problemDataClone);
+ ensembleSolution.AddRegressionSolutions(solutions.Value);
+
+ aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+ }
+ List flattenedResults = new List();
+ CollectResultsRecursively("", aggregatedResults, flattenedResults);
+ return flattenedResults;
+ }
+
+ private IEnumerable ExtractAndAggregateClassificationSolutions(IEnumerable> resultCollections) {
+ Dictionary> resultSolutions = new Dictionary>();
+ foreach (var result in resultCollections) {
+ var classificationSolution = result.Value as IClassificationSolution;
+ if (classificationSolution != null) {
+ if (resultSolutions.ContainsKey(result.Key)) {
+ resultSolutions[result.Key].Add(classificationSolution);
+ } else {
+ resultSolutions.Add(result.Key, new List() { classificationSolution });
+ }
+ }
+ }
+ var aggregatedResults = new List();
+ foreach (KeyValuePair> solutions in resultSolutions) {
+ // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
+ var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
+ var dataset = (Dataset)Problem.ProblemData.Dataset;
+ if (ShuffleSamples.Value) {
+ var random = new FastRandom(seed);
+ dataset = dataset.Shuffle(random);
+ }
+ var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
+ // set partitions of problem data clone correctly
+ problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+ problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+ // clone models
+ var ensembleSolution = new ClassificationEnsembleSolution(problemDataClone);
+ ensembleSolution.AddClassificationSolutions(solutions.Value);
+
+ aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+ }
+ List flattenedResults = new List();
+ CollectResultsRecursively("", aggregatedResults, flattenedResults);
+ return flattenedResults;
+ }
+
+ private void CollectResultsRecursively(string path, IEnumerable results, IList flattenedResults) {
+ foreach (IResult result in results) {
+ flattenedResults.Add(new Result(path + result.Name, result.Value));
+ ResultCollection childCollection = result.Value as ResultCollection;
+ if (childCollection != null) {
+ CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
+ }
+ }
+ }
+
+ private static IEnumerable ExtractAndAggregateResults(IEnumerable> results)
+ where T : class, IItem, new() {
+ Dictionary> resultValues = new Dictionary>();
+ foreach (var resultValue in results.Where(r => r.Value.GetType() == typeof(T))) {
+ if (!resultValues.ContainsKey(resultValue.Key))
+ resultValues[resultValue.Key] = new List();
+ resultValues[resultValue.Key].Add(ConvertToDouble(resultValue.Value));
+ }
+
+ DoubleValue doubleValue;
+ if (typeof(T) == typeof(PercentValue))
+ doubleValue = new PercentValue();
+ else if (typeof(T) == typeof(DoubleValue))
+ doubleValue = new DoubleValue();
+ else if (typeof(T) == typeof(IntValue))
+ doubleValue = new DoubleValue();
+ else
+ throw new NotSupportedException();
+
+ List aggregatedResults = new List();
+ foreach (KeyValuePair> resultValue in resultValues) {
+ doubleValue.Value = resultValue.Value.Average();
+ aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
+ doubleValue.Value = resultValue.Value.StandardDeviation();
+ aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
+ }
+ return aggregatedResults;
+ }
+
+ private static double ConvertToDouble(IItem item) {
+ if (item is DoubleValue) return ((DoubleValue)item).Value;
+ else if (item is IntValue) return ((IntValue)item).Value;
+ else throw new NotSupportedException("Could not convert any item type to double");
+ }
+ #endregion
+
+ #region events
+ private void RegisterEvents() {
+ Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
+ RegisterClonedAlgorithmsEvents();
+ }
+ private void Folds_ValueChanged(object sender, EventArgs e) {
+ if (ExecutionState != ExecutionState.Prepared)
+ throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
+ }
+
+
+ #region template algorithms events
+ public event EventHandler AlgorithmChanged;
+ private void OnAlgorithmChanged() {
+ EventHandler handler = AlgorithmChanged;
+ if (handler != null) handler(this, EventArgs.Empty);
+ OnProblemChanged();
+ if (Problem == null) ExecutionState = ExecutionState.Stopped;
+ }
+ private void RegisterAlgorithmEvents() {
+ algorithm.ProblemChanged += new EventHandler(Algorithm_ProblemChanged);
+ algorithm.ExecutionStateChanged += new EventHandler(Algorithm_ExecutionStateChanged);
+ if (Problem != null) {
+ Problem.Reset += new EventHandler(Problem_Reset);
+ }
+ }
+ private void DeregisterAlgorithmEvents() {
+ algorithm.ProblemChanged -= new EventHandler(Algorithm_ProblemChanged);
+ algorithm.ExecutionStateChanged -= new EventHandler(Algorithm_ExecutionStateChanged);
+ if (Problem != null) {
+ Problem.Reset -= new EventHandler(Problem_Reset);
+ }
+ }
+ private void Algorithm_ProblemChanged(object sender, EventArgs e) {
+ if (algorithm.Problem != null && !(algorithm.Problem is IDataAnalysisProblem)) {
+ algorithm.Problem = problem;
+ throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
+ }
+ if (problem != null) problem.Reset -= new EventHandler(Problem_Reset);
+ problem = (IDataAnalysisProblem)algorithm.Problem;
+ if (problem != null) problem.Reset += new EventHandler(Problem_Reset);
+ OnProblemChanged();
+ }
+ public event EventHandler ProblemChanged;
+ private void OnProblemChanged() {
+ EventHandler handler = ProblemChanged;
+ if (handler != null) handler(this, EventArgs.Empty);
+ ConfigureProblem();
+ }
+ private void Problem_Reset(object sender, EventArgs e) {
+ ConfigureProblem();
+ }
+ private void ConfigureProblem() {
+ SamplesStart.Value = 0;
+ if (Problem != null) {
+ SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
+
+ DataAnalysisProblemData problemData = Problem.ProblemData as DataAnalysisProblemData;
+ if (problemData != null) {
+ problemData.TrainingPartitionParameter.Hidden = true;
+ problemData.TestPartitionParameter.Hidden = true;
+ }
+ ISymbolicDataAnalysisProblem symbolicProblem = Problem as ISymbolicDataAnalysisProblem;
+ if (symbolicProblem != null) {
+ symbolicProblem.FitnessCalculationPartitionParameter.Hidden = true;
+ symbolicProblem.FitnessCalculationPartition.Start = SamplesStart.Value;
+ symbolicProblem.FitnessCalculationPartition.End = SamplesEnd.Value;
+ symbolicProblem.ValidationPartitionParameter.Hidden = true;
+ symbolicProblem.ValidationPartition.Start = 0;
+ symbolicProblem.ValidationPartition.End = 0;
+ }
+ } else
+ SamplesEnd.Value = 0;
+ }
+
+ private void Algorithm_ExecutionStateChanged(object sender, EventArgs e) {
+ switch (Algorithm.ExecutionState) {
+ case ExecutionState.Prepared:
+ OnPrepared();
+ break;
+ case ExecutionState.Started: throw new InvalidOperationException("Algorithm template can not be started.");
+ case ExecutionState.Paused: throw new InvalidOperationException("Algorithm template can not be paused.");
+ case ExecutionState.Stopped:
+ OnStopped();
+ break;
+ }
+ }
+ #endregion
+
+ #region clonedAlgorithms events
+ private void RegisterClonedAlgorithmsEvents() {
+ clonedAlgorithms.ItemsAdded += new CollectionItemsChangedEventHandler(ClonedAlgorithms_ItemsAdded);
+ clonedAlgorithms.ItemsRemoved += new CollectionItemsChangedEventHandler(ClonedAlgorithms_ItemsRemoved);
+ clonedAlgorithms.CollectionReset += new CollectionItemsChangedEventHandler(ClonedAlgorithms_CollectionReset);
+ foreach (IAlgorithm algorithm in clonedAlgorithms)
+ RegisterClonedAlgorithmEvents(algorithm);
+ }
+ private void DeregisterClonedAlgorithmsEvents() {
+ clonedAlgorithms.ItemsAdded -= new CollectionItemsChangedEventHandler(ClonedAlgorithms_ItemsAdded);
+ clonedAlgorithms.ItemsRemoved -= new CollectionItemsChangedEventHandler(ClonedAlgorithms_ItemsRemoved);
+ clonedAlgorithms.CollectionReset -= new CollectionItemsChangedEventHandler(ClonedAlgorithms_CollectionReset);
+ foreach (IAlgorithm algorithm in clonedAlgorithms)
+ DeregisterClonedAlgorithmEvents(algorithm);
+ }
+ private void ClonedAlgorithms_ItemsAdded(object sender, CollectionItemsChangedEventArgs e) {
+ foreach (IAlgorithm algorithm in e.Items)
+ RegisterClonedAlgorithmEvents(algorithm);
+ }
+ private void ClonedAlgorithms_ItemsRemoved(object sender, CollectionItemsChangedEventArgs e) {
+ foreach (IAlgorithm algorithm in e.Items)
+ DeregisterClonedAlgorithmEvents(algorithm);
+ }
+ private void ClonedAlgorithms_CollectionReset(object sender, CollectionItemsChangedEventArgs e) {
+ foreach (IAlgorithm algorithm in e.OldItems)
+ DeregisterClonedAlgorithmEvents(algorithm);
+ foreach (IAlgorithm algorithm in e.Items)
+ RegisterClonedAlgorithmEvents(algorithm);
+ }
+ private void RegisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+ algorithm.ExceptionOccurred += new EventHandler>(ClonedAlgorithm_ExceptionOccurred);
+ algorithm.ExecutionTimeChanged += new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+ algorithm.Started += new EventHandler(ClonedAlgorithm_Started);
+ algorithm.Paused += new EventHandler(ClonedAlgorithm_Paused);
+ algorithm.Stopped += new EventHandler(ClonedAlgorithm_Stopped);
+ }
+ private void DeregisterClonedAlgorithmEvents(IAlgorithm algorithm) {
+ algorithm.ExceptionOccurred -= new EventHandler>(ClonedAlgorithm_ExceptionOccurred);
+ algorithm.ExecutionTimeChanged -= new EventHandler(ClonedAlgorithm_ExecutionTimeChanged);
+ algorithm.Started -= new EventHandler(ClonedAlgorithm_Started);
+ algorithm.Paused -= new EventHandler(ClonedAlgorithm_Paused);
+ algorithm.Stopped -= new EventHandler(ClonedAlgorithm_Stopped);
+ }
+ private void ClonedAlgorithm_ExceptionOccurred(object sender, EventArgs e) {
+ Pause();
+ OnExceptionOccurred(e.Value);
+ }
+ private void ClonedAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) {
+ OnExecutionTimeChanged();
+ }
+
+ private readonly object locker = new object();
+ private readonly object resultLocker = new object();
+ private void ClonedAlgorithm_Started(object sender, EventArgs e) {
+ IAlgorithm algorithm = sender as IAlgorithm;
+ lock (resultLocker) {
+ if (algorithm != null && !results.ContainsKey(algorithm.Name))
+ results.Add(new Result(algorithm.Name, "Contains results for the specific fold.", algorithm.Results));
+ }
+ }
+
+ private void ClonedAlgorithm_Paused(object sender, EventArgs e) {
+ lock (locker) {
+ availableWorkers.Release();
+ if (clonedAlgorithms.All(alg => alg.ExecutionState != ExecutionState.Started)) {
+ OnPaused();
+ allAlgorithmsFinished.Set();
+ }
+ }
+ }
+
+ private void ClonedAlgorithm_Stopped(object sender, EventArgs e) {
+ lock (locker) {
+ // if the algorithm was in paused state, its worker has already been released
+ if (availableWorkers.CurrentCount < NumberOfWorkers.Value)
+ availableWorkers.Release();
+ if (clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Stopped)) {
+ OnStopped();
+ allAlgorithmsFinished.Set();
+ } else if (stopPending && clonedAlgorithms.All(alg => alg.ExecutionState == ExecutionState.Prepared || alg.ExecutionState == ExecutionState.Stopped)) {
+ OnStopped();
+ allAlgorithmsFinished.Set();
+ }
+ }
+ }
+ #endregion
+ #endregion
+
+ #region event firing
+ public event EventHandler ExecutionStateChanged;
+ private void OnExecutionStateChanged() {
+ EventHandler handler = ExecutionStateChanged;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ public event EventHandler ExecutionTimeChanged;
+ private void OnExecutionTimeChanged() {
+ EventHandler handler = ExecutionTimeChanged;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ public event EventHandler Prepared;
+ private void OnPrepared() {
+ ExecutionState = ExecutionState.Prepared;
+ EventHandler handler = Prepared;
+ if (handler != null) handler(this, EventArgs.Empty);
+ OnExecutionTimeChanged();
+ }
+ public event EventHandler Started;
+ private void OnStarted() {
+ startPending = false;
+ ExecutionState = ExecutionState.Started;
+ EventHandler handler = Started;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ public event EventHandler Paused;
+ private void OnPaused() {
+ pausePending = false;
+ ExecutionState = ExecutionState.Paused;
+ EventHandler handler = Paused;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ public event EventHandler Stopped;
+ private void OnStopped() {
+ stopPending = false;
+ Dictionary collectedResults = new Dictionary();
+ AggregateResultValues(collectedResults);
+ results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast().ToArray());
+ clonedAlgorithms.Clear();
+ runsCounter++;
+ runs.Add(new Run(string.Format("{0} Run {1}", Name, runsCounter), this));
+ ExecutionState = ExecutionState.Stopped;
+ EventHandler handler = Stopped;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ public event EventHandler> ExceptionOccurred;
+ private void OnExceptionOccurred(Exception exception) {
+ EventHandler> handler = ExceptionOccurred;
+ if (handler != null) handler(this, new EventArgs(exception));
+ }
+ public event EventHandler StoreAlgorithmInEachRunChanged;
+ private void OnStoreAlgorithmInEachRunChanged() {
+ EventHandler handler = StoreAlgorithmInEachRunChanged;
+ if (handler != null) handler(this, EventArgs.Empty);
+ }
+ #endregion
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/FixedDataAnalysisAlgorithm.cs (revision 15428)
@@ -0,0 +1,51 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ public abstract class FixedDataAnalysisAlgorithm : BasicAlgorithm where T : class, IDataAnalysisProblem {
+ #region Properties
+ public override Type ProblemType {
+ get { return typeof(T); }
+ }
+ public new T Problem {
+ get { return (T)base.Problem; }
+ set { base.Problem = value; }
+ }
+ #endregion
+
+ public override bool SupportsPause { get { return false; } }
+
+ [StorableConstructor]
+ protected FixedDataAnalysisAlgorithm(bool deserializing) : base(deserializing) { }
+ protected FixedDataAnalysisAlgorithm(FixedDataAnalysisAlgorithm original, Cloner cloner) : base(original, cloner) { }
+ public FixedDataAnalysisAlgorithm() : base() { }
+
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs (revision 15428)
@@ -0,0 +1,507 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * and the BEACON Center for the Study of Evolution in Action.
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Random;
+using HeuristicLab.Selection;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.MctsSymbolicRegression {
+ [Item("Gradient Boosting Machine Regression (GBM)",
+ "Gradient boosting for any regression base learner (e.g. MCTS symbolic regression)")]
+ [StorableClass]
+ [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 350)]
+ public class GradientBoostingRegressionAlgorithm : FixedDataAnalysisAlgorithm {
+
+ #region ParameterNames
+
+ private const string IterationsParameterName = "Iterations";
+ private const string NuParameterName = "Nu";
+ private const string MParameterName = "M";
+ private const string RParameterName = "R";
+ private const string RegressionAlgorithmParameterName = "RegressionAlgorithm";
+ private const string SeedParameterName = "Seed";
+ private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+ private const string CreateSolutionParameterName = "CreateSolution";
+ private const string StoreRunsParameterName = "StoreRuns";
+ private const string RegressionAlgorithmSolutionResultParameterName = "RegressionAlgorithmResult";
+
+ #endregion
+
+ #region ParameterProperties
+
+ public IFixedValueParameter IterationsParameter {
+ get { return (IFixedValueParameter)Parameters[IterationsParameterName]; }
+ }
+
+ public IFixedValueParameter NuParameter {
+ get { return (IFixedValueParameter)Parameters[NuParameterName]; }
+ }
+
+ public IFixedValueParameter RParameter {
+ get { return (IFixedValueParameter)Parameters[RParameterName]; }
+ }
+
+ public IFixedValueParameter MParameter {
+ get { return (IFixedValueParameter)Parameters[MParameterName]; }
+ }
+
+ // regression algorithms are currently: DataAnalysisAlgorithms, BasicAlgorithms and engine algorithms with no common interface
+ public IConstrainedValueParameter RegressionAlgorithmParameter {
+ get { return (IConstrainedValueParameter)Parameters[RegressionAlgorithmParameterName]; }
+ }
+
+ public IFixedValueParameter RegressionAlgorithmSolutionResultParameter {
+ get { return (IFixedValueParameter)Parameters[RegressionAlgorithmSolutionResultParameterName]; }
+ }
+
+ public IFixedValueParameter SeedParameter {
+ get { return (IFixedValueParameter)Parameters[SeedParameterName]; }
+ }
+
+ public FixedValueParameter SetSeedRandomlyParameter {
+ get { return (FixedValueParameter)Parameters[SetSeedRandomlyParameterName]; }
+ }
+
+ public IFixedValueParameter CreateSolutionParameter {
+ get { return (IFixedValueParameter)Parameters[CreateSolutionParameterName]; }
+ }
+ public IFixedValueParameter StoreRunsParameter {
+ get { return (IFixedValueParameter)Parameters[StoreRunsParameterName]; }
+ }
+
+ #endregion
+
+ #region Properties
+
+ public int Iterations {
+ get { return IterationsParameter.Value.Value; }
+ set { IterationsParameter.Value.Value = value; }
+ }
+
+ public int Seed {
+ get { return SeedParameter.Value.Value; }
+ set { SeedParameter.Value.Value = value; }
+ }
+
+ public bool SetSeedRandomly {
+ get { return SetSeedRandomlyParameter.Value.Value; }
+ set { SetSeedRandomlyParameter.Value.Value = value; }
+ }
+
+ public double Nu {
+ get { return NuParameter.Value.Value; }
+ set { NuParameter.Value.Value = value; }
+ }
+
+ public double R {
+ get { return RParameter.Value.Value; }
+ set { RParameter.Value.Value = value; }
+ }
+
+ public double M {
+ get { return MParameter.Value.Value; }
+ set { MParameter.Value.Value = value; }
+ }
+
+ public bool CreateSolution {
+ get { return CreateSolutionParameter.Value.Value; }
+ set { CreateSolutionParameter.Value.Value = value; }
+ }
+
+ public bool StoreRuns {
+ get { return StoreRunsParameter.Value.Value; }
+ set { StoreRunsParameter.Value.Value = value; }
+ }
+
+ public IAlgorithm RegressionAlgorithm {
+ get { return RegressionAlgorithmParameter.Value; }
+ }
+
+ public string RegressionAlgorithmResult {
+ get { return RegressionAlgorithmSolutionResultParameter.Value.Value; }
+ set { RegressionAlgorithmSolutionResultParameter.Value.Value = value; }
+ }
+
+ #endregion
+
+ [StorableConstructor]
+ protected GradientBoostingRegressionAlgorithm(bool deserializing)
+ : base(deserializing) {
+ }
+
+ protected GradientBoostingRegressionAlgorithm(GradientBoostingRegressionAlgorithm original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new GradientBoostingRegressionAlgorithm(this, cloner);
+ }
+
+ public GradientBoostingRegressionAlgorithm() {
+ Problem = new RegressionProblem(); // default problem
+ var osgp = CreateOSGP();
+ var regressionAlgs = new ItemSet(new IAlgorithm[] {
+ new RandomForestRegression(),
+ osgp,
+ });
+ foreach (var alg in regressionAlgs) alg.Prepare();
+
+
+ Parameters.Add(new FixedValueParameter(IterationsParameterName,
+ "Number of iterations", new IntValue(100)));
+ Parameters.Add(new FixedValueParameter(SeedParameterName,
+ "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
+ Parameters.Add(new FixedValueParameter(SetSeedRandomlyParameterName,
+ "True if the random seed should be set to a random value, otherwise false.", new BoolValue(true)));
+ Parameters.Add(new FixedValueParameter(NuParameterName,
+ "The learning rate nu when updating predictions in GBM (0 < nu <= 1)", new DoubleValue(0.5)));
+ Parameters.Add(new FixedValueParameter(RParameterName,
+ "The fraction of rows that are sampled randomly for the base learner in each iteration (0 < r <= 1)",
+ new DoubleValue(1)));
+ Parameters.Add(new FixedValueParameter(MParameterName,
+ "The fraction of variables that are sampled randomly for the base learner in each iteration (0 < m <= 1)",
+ new DoubleValue(0.5)));
+ Parameters.Add(new ConstrainedValueParameter(RegressionAlgorithmParameterName,
+ "The regression algorithm to use as a base learner", regressionAlgs, osgp));
+ Parameters.Add(new FixedValueParameter(RegressionAlgorithmSolutionResultParameterName,
+ "The name of the solution produced by the regression algorithm", new StringValue("Solution")));
+ Parameters[RegressionAlgorithmSolutionResultParameterName].Hidden = true;
+ Parameters.Add(new FixedValueParameter(CreateSolutionParameterName,
+ "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
+ Parameters[CreateSolutionParameterName].Hidden = true;
+ Parameters.Add(new FixedValueParameter(StoreRunsParameterName,
+ "Flag that indicates if the results of the individual runs should be stored for detailed analysis", new BoolValue(false)));
+ Parameters[StoreRunsParameterName].Hidden = true;
+ }
+
+ protected override void Run(CancellationToken cancellationToken) {
+ // Set up the algorithm
+ if (SetSeedRandomly) Seed = new System.Random().Next();
+ var rand = new MersenneTwister((uint)Seed);
+
+ // Set up the results display
+ var iterations = new IntValue(0);
+ Results.Add(new Result("Iterations", iterations));
+
+ var table = new DataTable("Qualities");
+ table.Rows.Add(new DataRow("R² (train)"));
+ table.Rows.Add(new DataRow("R² (test)"));
+ Results.Add(new Result("Qualities", table));
+ var curLoss = new DoubleValue();
+ var curTestLoss = new DoubleValue();
+ Results.Add(new Result("R² (train)", curLoss));
+ Results.Add(new Result("R² (test)", curTestLoss));
+ var runCollection = new RunCollection();
+ if (StoreRuns)
+ Results.Add(new Result("Runs", runCollection));
+
+ // init
+ var problemData = Problem.ProblemData;
+ var targetVarName = problemData.TargetVariable;
+ var activeVariables = problemData.AllowedInputVariables.Concat(new string[] { problemData.TargetVariable });
+ var modifiableDataset = new ModifiableDataset(
+ activeVariables,
+ activeVariables.Select(v => problemData.Dataset.GetDoubleValues(v).ToList()));
+
+ var trainingRows = problemData.TrainingIndices;
+ var testRows = problemData.TestIndices;
+ var yPred = new double[trainingRows.Count()];
+ var yPredTest = new double[testRows.Count()];
+ var y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+ var curY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+ var yTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+ var curYTest = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToArray();
+ var nu = Nu;
+ var mVars = (int)Math.Ceiling(M * problemData.AllowedInputVariables.Count());
+ var rRows = (int)Math.Ceiling(R * problemData.TrainingIndices.Count());
+ var alg = RegressionAlgorithm;
+ List models = new List();
+ try {
+
+ // Loop until iteration limit reached or canceled.
+ for (int i = 0; i < Iterations; i++) {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ modifiableDataset.RemoveVariable(targetVarName);
+ modifiableDataset.AddVariable(targetVarName, curY.Concat(curYTest));
+
+ SampleTrainingData(rand, modifiableDataset, rRows, problemData.Dataset, curY, problemData.TargetVariable, problemData.TrainingIndices); // all training indices from the original problem data are allowed
+ var modifiableProblemData = new RegressionProblemData(modifiableDataset,
+ problemData.AllowedInputVariables.SampleRandomWithoutRepetition(rand, mVars),
+ problemData.TargetVariable);
+ modifiableProblemData.TrainingPartition.Start = 0;
+ modifiableProblemData.TrainingPartition.End = rRows;
+ modifiableProblemData.TestPartition.Start = problemData.TestPartition.Start;
+ modifiableProblemData.TestPartition.End = problemData.TestPartition.End;
+
+ if (!TrySetProblemData(alg, modifiableProblemData))
+ throw new NotSupportedException("The algorithm cannot be used with GBM.");
+
+ IRegressionModel model;
+ IRun run;
+
+ // try to find a model. The algorithm might fail to produce a model. In this case we just retry until the iterations are exhausted
+ if (TryExecute(alg, rand.Next(), RegressionAlgorithmResult, out model, out run)) {
+ int row = 0;
+ // update predictions for training and test
+ // update new targets (in the case of squared error loss we simply use negative residuals)
+ foreach (var pred in model.GetEstimatedValues(problemData.Dataset, trainingRows)) {
+ yPred[row] = yPred[row] + nu * pred;
+ curY[row] = y[row] - yPred[row];
+ row++;
+ }
+ row = 0;
+ foreach (var pred in model.GetEstimatedValues(problemData.Dataset, testRows)) {
+ yPredTest[row] = yPredTest[row] + nu * pred;
+ curYTest[row] = yTest[row] - yPredTest[row];
+ row++;
+ }
+ // determine quality
+ OnlineCalculatorError error;
+ var trainR = OnlinePearsonsRCalculator.Calculate(yPred, y, out error);
+ var testR = OnlinePearsonsRCalculator.Calculate(yPredTest, yTest, out error);
+
+ // iteration results
+ curLoss.Value = error == OnlineCalculatorError.None ? trainR * trainR : 0.0;
+ curTestLoss.Value = error == OnlineCalculatorError.None ? testR * testR : 0.0;
+
+ models.Add(model);
+
+
+ }
+
+ if (StoreRuns)
+ runCollection.Add(run);
+ table.Rows["R² (train)"].Values.Add(curLoss.Value);
+ table.Rows["R² (test)"].Values.Add(curTestLoss.Value);
+ iterations.Value = i + 1;
+ }
+
+ // produce solution
+ if (CreateSolution) {
+ // when all our models are symbolic models we can easily combine them to a single model
+ if (models.All(m => m is ISymbolicRegressionModel)) {
+ Results.Add(new Result("Solution", CreateSymbolicSolution(models, Nu, (IRegressionProblemData)problemData.Clone())));
+ }
+ // just produce an ensemble solution for now (TODO: correct scaling or linear regression for ensemble model weights)
+
+ var ensembleSolution = CreateEnsembleSolution(models, (IRegressionProblemData)problemData.Clone());
+ Results.Add(new Result("EnsembleSolution", ensembleSolution));
+ }
+ }
+ finally {
+ // reset everything
+ alg.Prepare(true);
+ }
+ }
+
+ private static IRegressionEnsembleSolution CreateEnsembleSolution(List models,
+ IRegressionProblemData problemData) {
+ var rows = problemData.TrainingPartition.Size;
+ var features = models.Count;
+ double[,] inputMatrix = new double[rows, features + 1];
+ //add model estimates
+ for (int m = 0; m < models.Count; m++) {
+ var model = models[m];
+ var estimates = model.GetEstimatedValues(problemData.Dataset, problemData.TrainingIndices);
+ int estimatesCounter = 0;
+ foreach (var estimate in estimates) {
+ inputMatrix[estimatesCounter, m] = estimate;
+ estimatesCounter++;
+ }
+ }
+
+ //add target
+ var targets = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices);
+ int targetCounter = 0;
+ foreach (var target in targets) {
+ inputMatrix[targetCounter, models.Count] = target;
+ targetCounter++;
+ }
+
+ alglib.linearmodel lm = new alglib.linearmodel();
+ alglib.lrreport ar = new alglib.lrreport();
+ double[] coefficients;
+ int retVal = 1;
+ alglib.lrbuildz(inputMatrix, rows, features, out retVal, out lm, out ar);
+ if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+
+ alglib.lrunpack(lm, out coefficients, out features);
+
+ var ensembleModel = new RegressionEnsembleModel(models, coefficients.Take(models.Count)) { AverageModelEstimates = false };
+ var ensembleSolution = (IRegressionEnsembleSolution)ensembleModel.CreateRegressionSolution(problemData);
+ return ensembleSolution;
+ }
+
+
+ private IAlgorithm CreateOSGP() {
+ // configure strict osgp
+ var alg = new OffspringSelectionGeneticAlgorithm.OffspringSelectionGeneticAlgorithm();
+ var prob = new SymbolicRegressionSingleObjectiveProblem();
+ prob.MaximumSymbolicExpressionTreeDepth.Value = 7;
+ prob.MaximumSymbolicExpressionTreeLength.Value = 15;
+ alg.Problem = prob;
+ alg.SuccessRatio.Value = 1.0;
+ alg.ComparisonFactorLowerBound.Value = 1.0;
+ alg.ComparisonFactorUpperBound.Value = 1.0;
+ alg.MutationProbability.Value = 0.15;
+ alg.PopulationSize.Value = 200;
+ alg.MaximumSelectionPressure.Value = 100;
+ alg.MaximumEvaluatedSolutions.Value = 20000;
+ alg.SelectorParameter.Value = alg.SelectorParameter.ValidValues.OfType().First();
+ alg.MutatorParameter.Value = alg.MutatorParameter.ValidValues.OfType().First();
+ alg.StoreAlgorithmInEachRun = false;
+ return alg;
+ }
+
+ private void SampleTrainingData(MersenneTwister rand, ModifiableDataset ds, int rRows,
+ IDataset sourceDs, double[] curTarget, string targetVarName, IEnumerable trainingIndices) {
+ var selectedRows = trainingIndices.SampleRandomWithoutRepetition(rand, rRows).ToArray();
+ int t = 0;
+ object[] srcRow = new object[ds.Columns];
+ var varNames = ds.DoubleVariables.ToArray();
+ foreach (var r in selectedRows) {
+ // take all values from the original dataset
+ for (int c = 0; c < srcRow.Length; c++) {
+ var col = sourceDs.GetReadOnlyDoubleValues(varNames[c]);
+ srcRow[c] = col[r];
+ }
+ ds.ReplaceRow(t, srcRow);
+ // but use the updated target values
+ ds.SetVariableValue(curTarget[r], targetVarName, t);
+ t++;
+ }
+ }
+
+ private static ISymbolicRegressionSolution CreateSymbolicSolution(List models, double nu, IRegressionProblemData problemData) {
+ var symbModels = models.OfType();
+ var lowerLimit = symbModels.Min(m => m.LowerEstimationLimit);
+ var upperLimit = symbModels.Max(m => m.UpperEstimationLimit);
+ var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+ var progRootNode = new ProgramRootSymbol().CreateTreeNode();
+ var startNode = new StartSymbol().CreateTreeNode();
+
+ var addNode = new Addition().CreateTreeNode();
+ var mulNode = new Multiplication().CreateTreeNode();
+ var scaleNode = (ConstantTreeNode)new Constant().CreateTreeNode(); // all models are scaled using the same nu
+ scaleNode.Value = nu;
+
+ foreach (var m in symbModels) {
+ var relevantPart = m.SymbolicExpressionTree.Root.GetSubtree(0).GetSubtree(0); // skip root and start
+ addNode.AddSubtree((ISymbolicExpressionTreeNode)relevantPart.Clone());
+ }
+
+ mulNode.AddSubtree(addNode);
+ mulNode.AddSubtree(scaleNode);
+ startNode.AddSubtree(mulNode);
+ progRootNode.AddSubtree(startNode);
+ var t = new SymbolicExpressionTree(progRootNode);
+ var combinedModel = new SymbolicRegressionModel(problemData.TargetVariable, t, interpreter, lowerLimit, upperLimit);
+ var sol = new SymbolicRegressionSolution(combinedModel, problemData);
+ return sol;
+ }
+
+ private static bool TrySetProblemData(IAlgorithm alg, IRegressionProblemData problemData) {
+ var prob = alg.Problem as IRegressionProblem;
+ // there is already a problem and it is compatible -> just set problem data
+ if (prob != null) {
+ prob.ProblemDataParameter.Value = problemData;
+ return true;
+ } else return false;
+ }
+
+ private static bool TryExecute(IAlgorithm alg, int seed, string regressionAlgorithmResultName, out IRegressionModel model, out IRun run) {
+ model = null;
+ SetSeed(alg, seed);
+ using (var wh = new AutoResetEvent(false)) {
+ Exception ex = null;
+ EventHandler> handler = (sender, args) => {
+ ex = args.Value;
+ wh.Set();
+ };
+ EventHandler handler2 = (sender, args) => wh.Set();
+ alg.ExceptionOccurred += handler;
+ alg.Stopped += handler2;
+ try {
+ alg.Prepare();
+ alg.Start();
+ wh.WaitOne();
+
+ if (ex != null) throw new AggregateException(ex);
+ run = alg.Runs.Last();
+ alg.Runs.Clear();
+ var sols = alg.Results.Select(r => r.Value).OfType();
+ if (!sols.Any()) return false;
+ var sol = sols.First();
+ if (sols.Skip(1).Any()) {
+ // more than one solution => use regressionAlgorithmResult
+ if (alg.Results.ContainsKey(regressionAlgorithmResultName)) {
+ sol = (IRegressionSolution)alg.Results[regressionAlgorithmResultName].Value;
+ }
+ }
+ var symbRegSol = sol as SymbolicRegressionSolution;
+ // only accept symb reg solutions that do not hit the estimation limits
+ // NaN evaluations would not be critical but are problematic if we want to combine all symbolic models into a single symbolic model
+ if (symbRegSol == null ||
+ (symbRegSol.TrainingLowerEstimationLimitHits == 0 && symbRegSol.TrainingUpperEstimationLimitHits == 0 &&
+ symbRegSol.TestLowerEstimationLimitHits == 0 && symbRegSol.TestUpperEstimationLimitHits == 0) &&
+ symbRegSol.TrainingNaNEvaluations == 0 && symbRegSol.TestNaNEvaluations == 0) {
+ model = sol.Model;
+ }
+ }
+ finally {
+ alg.ExceptionOccurred -= handler;
+ alg.Stopped -= handler2;
+ }
+ }
+ return model != null;
+ }
+
+ private static void SetSeed(IAlgorithm alg, int seed) {
+ // no common interface for algs that use a PRNG -> use naming convention to set seed
+ var paramItem = alg as IParameterizedItem;
+
+ if (paramItem.Parameters.ContainsKey("SetSeedRandomly")) {
+ ((BoolValue)paramItem.Parameters["SetSeedRandomly"].ActualValue).Value = false;
+ ((IntValue)paramItem.Parameters["Seed"].ActualValue).Value = seed;
+ } else {
+ throw new ArgumentException("Base learner does not have a seed parameter (algorithm {0})", alg.Name);
+ }
+
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceConst.cs (revision 15428)
@@ -0,0 +1,98 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceConst",
+ Description = "Constant covariance function for Gaussian processes.")]
+ public sealed class CovarianceConst : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ [StorableConstructor]
+ private CovarianceConst(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceConst(CovarianceConst original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceConst()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale of the constant covariance function."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceConst(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return HasFixedScaleParameter ? 0 : 1;
+ }
+
+ public void SetParameter(double[] p) {
+ double scale;
+ GetParameterValues(p, out scale);
+ ScaleParameter.Value = new DoubleValue(scale);
+ }
+
+ private void GetParameterValues(double[] p, out double scale) {
+ int c = 0;
+ // gather parameter values
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceConst", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double scale;
+ GetParameterValues(p, out scale);
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => scale;
+ cov.CrossCovariance = (x, xt, i, j) => scale;
+ if (HasFixedScaleParameter) {
+ cov.CovarianceGradient = (x, i, j) => new double[0];
+ } else {
+ cov.CovarianceGradient = (x, i, j) => new[] { 2.0 * scale };
+ }
+ return cov;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinear.cs (revision 15428)
@@ -0,0 +1,62 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceLinear", Description = "Linear covariance function for Gaussian processes.")]
+ public sealed class CovarianceLinear : Item, ICovarianceFunction {
+ [StorableConstructor]
+ private CovarianceLinear(bool deserializing) : base(deserializing) { }
+ private CovarianceLinear(CovarianceLinear original, Cloner cloner)
+ : base(original, cloner) {
+ }
+ public CovarianceLinear()
+ : base() {
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceLinear(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return 0;
+ }
+
+ public void SetParameter(double[] p) {
+ if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ if (p.Length > 0) throw new ArgumentException("No parameters are allowed for the linear covariance function.");
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, columnIndices, 1.0);
+ cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, columnIndices, 1.0);
+ cov.CovarianceGradient = (x, i, j) => new double[0];
+ return cov;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceLinearArd.cs (revision 15428)
@@ -0,0 +1,109 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceLinearArd",
+ Description = "Linear covariance function with automatic relevance determination for Gaussian processes.")]
+ public sealed class CovarianceLinearArd : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter InverseLengthParameter {
+ get { return (IValueParameter)Parameters["InverseLength"]; }
+ }
+ private bool HasFixedInverseLengthParameter {
+ get { return InverseLengthParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceLinearArd(bool deserializing) : base(deserializing) { }
+ private CovarianceLinearArd(CovarianceLinearArd original, Cloner cloner)
+ : base(original, cloner) {
+ }
+ public CovarianceLinearArd()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("InverseLength",
+ "The inverse length parameter for ARD."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceLinearArd(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ if (HasFixedInverseLengthParameter)
+ return 0;
+ else
+ return numberOfVariables;
+ }
+
+ public void SetParameter(double[] p) {
+ double[] inverseLength;
+ GetParameterValues(p, out inverseLength);
+ InverseLengthParameter.Value = new DoubleArray(inverseLength);
+ }
+
+ private void GetParameterValues(double[] p, out double[] inverseLength) {
+ // gather parameter values
+ if (HasFixedInverseLengthParameter) {
+ inverseLength = InverseLengthParameter.Value.ToArray();
+ } else {
+ inverseLength = p.Select(e => 1.0 / Math.Exp(e)).ToArray();
+ }
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double[] inverseLength;
+ GetParameterValues(p, out inverseLength);
+ var fixedInverseLength = HasFixedInverseLengthParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => Util.ScalarProd(x, i, j, inverseLength, columnIndices);
+ cov.CrossCovariance = (x, xt, i, j) => Util.ScalarProd(x, i, xt, j, inverseLength, columnIndices);
+ if (fixedInverseLength)
+ cov.CovarianceGradient = (x, i, j) => new double[0];
+ else
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, inverseLength, columnIndices);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, double[] inverseLength, int[] columnIndices) {
+ int k = 0;
+ var g = new List(columnIndices.Length);
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var columnIndex = columnIndices[c];
+ g.Add(-2.0 * x[i, columnIndex] * x[j, columnIndex] * inverseLength[k] * inverseLength[k]);
+ k++;
+ }
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMask.cs (revision 15428)
@@ -0,0 +1,79 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceMask",
+ Description = "Masking covariance function for dimension selection can be used to apply a covariance function only on certain input dimensions.")]
+ public sealed class CovarianceMask : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter SelectedDimensionsParameter {
+ get { return (IValueParameter)Parameters["SelectedDimensions"]; }
+ }
+ public IValueParameter CovarianceFunctionParameter {
+ get { return (IValueParameter)Parameters["CovarianceFunction"]; }
+ }
+
+ [StorableConstructor]
+ private CovarianceMask(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceMask(CovarianceMask original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceMask()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("SelectedDimensions", "The dimensions on which the specified covariance function should be applied to."));
+ Parameters.Add(new ValueParameter("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceMask(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ if (SelectedDimensionsParameter.Value == null) return CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+ else return CovarianceFunctionParameter.Value.GetNumberOfParameters(SelectedDimensionsParameter.Value.Length);
+ }
+
+ public void SetParameter(double[] p) {
+ CovarianceFunctionParameter.Value.SetParameter(p);
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ var cov = CovarianceFunctionParameter.Value;
+ var selectedDimensions = SelectedDimensionsParameter.Value;
+
+ return cov.GetParameterizedCovarianceFunction(p, selectedDimensions.Intersect(columnIndices).ToArray());
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceMaternIso.cs (revision 15428)
@@ -0,0 +1,169 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceMaternIso",
+ Description = "Matern covariance function for Gaussian processes.")]
+ public sealed class CovarianceMaternIso : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter InverseLengthParameter {
+ get { return (IValueParameter)Parameters["InverseLength"]; }
+ }
+
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IConstrainedValueParameter DParameter {
+ get { return (IConstrainedValueParameter)Parameters["D"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ private bool HasFixedInverseLengthParameter {
+ get { return InverseLengthParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceMaternIso(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceMaternIso(CovarianceMaternIso original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceMaternIso()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("InverseLength", "The inverse length parameter of the isometric Matern covariance function."));
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter of the isometric Matern covariance function."));
+ var validDValues = new ItemSet();
+ validDValues.Add((IntValue)new IntValue(1).AsReadOnly());
+ validDValues.Add((IntValue)new IntValue(3).AsReadOnly());
+ validDValues.Add((IntValue)new IntValue(5).AsReadOnly());
+ Parameters.Add(new ConstrainedValueParameter("D", "The d parameter (allowed values: 1, 3, or 5) of the isometric Matern covariance function.", validDValues, validDValues.First()));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceMaternIso(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return
+ (HasFixedInverseLengthParameter ? 0 : 1) +
+ (HasFixedScaleParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double inverseLength, scale;
+ GetParameterValues(p, out scale, out inverseLength);
+ InverseLengthParameter.Value = new DoubleValue(inverseLength);
+ ScaleParameter.Value = new DoubleValue(scale);
+ }
+
+ private void GetParameterValues(double[] p, out double scale, out double inverseLength) {
+ // gather parameter values
+ int c = 0;
+ if (HasFixedInverseLengthParameter) {
+ inverseLength = InverseLengthParameter.Value.Value;
+ } else {
+ inverseLength = 1.0 / Math.Exp(p[c]);
+ c++;
+ }
+
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceMaternIso", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double inverseLength, scale;
+ int d = DParameter.Value.Value;
+ GetParameterValues(p, out scale, out inverseLength);
+ var fixedInverseLength = HasFixedInverseLengthParameter;
+ var fixedScale = HasFixedScaleParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double dist = i == j
+ ? 0.0
+ : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
+ return scale * m(d, dist);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double dist = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, Math.Sqrt(d) * inverseLength));
+ return scale * m(d, dist);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, d, scale, inverseLength, columnIndices, fixedInverseLength, fixedScale);
+ return cov;
+ }
+
+ private static double m(int d, double t) {
+ double f;
+ switch (d) {
+ case 1: { f = 1; break; }
+ case 3: { f = 1 + t; break; }
+ case 5: { f = 1 + t * (1 + t / 3.0); break; }
+ default: throw new InvalidOperationException();
+ }
+ return f * Math.Exp(-t);
+ }
+
+ private static double dm(int d, double t) {
+ double df;
+ switch (d) {
+ case 1: { df = 1; break; }
+ case 3: { df = t; break; }
+ case 5: { df = t * (1 + t) / 3.0; break; }
+ default: throw new InvalidOperationException();
+ }
+ return df * t * Math.Exp(-t);
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, int d, double scale, double inverseLength, int[] columnIndices,
+ bool fixedInverseLength, bool fixedScale) {
+ double dist = i == j
+ ? 0.0
+ : Math.Sqrt(Util.SqrDist(x, i, j, columnIndices, Math.Sqrt(d) * inverseLength));
+
+ var g = new List(2);
+ if (!fixedInverseLength) g.Add(scale * dm(d, dist));
+ if (!fixedScale) g.Add(2 * scale * m(d, dist));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNeuralNetwork.cs (revision 15428)
@@ -0,0 +1,162 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceNeuralNetwork",
+ Description = "Neural network covariance function for Gaussian processes.")]
+ public sealed class CovarianceNeuralNetwork : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter LengthParameter {
+ get { return (IValueParameter)Parameters["Length"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ private bool HasFixedLengthParameter {
+ get { return LengthParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceNeuralNetwork(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceNeuralNetwork(CovarianceNeuralNetwork original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceNeuralNetwork()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter."));
+ Parameters.Add(new OptionalValueParameter("Length", "The length parameter."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceNeuralNetwork(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return
+ (HasFixedScaleParameter ? 0 : 1) +
+ (HasFixedLengthParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double scale, length;
+ GetParameterValues(p, out scale, out length);
+ ScaleParameter.Value = new DoubleValue(scale);
+ LengthParameter.Value = new DoubleValue(length);
+ }
+
+
+ private void GetParameterValues(double[] p, out double scale, out double length) {
+ // gather parameter values
+ int c = 0;
+ if (HasFixedLengthParameter) {
+ length = LengthParameter.Value.Value;
+ } else {
+ length = Math.Exp(2 * p[c]);
+ c++;
+ }
+
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNeuralNetwork", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double length, scale;
+ GetParameterValues(p, out scale, out length);
+ var fixedLength = HasFixedLengthParameter;
+ var fixedScale = HasFixedScaleParameter;
+
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double sx = 1.0;
+ double s1 = 1.0;
+ double s2 = 1.0;
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var col = columnIndices[c];
+ sx += x[i, col] * x[j, col];
+ s1 += x[i, col] * x[i, col];
+ s2 += x[j, col] * x[j, col];
+ }
+
+ return (scale * Math.Asin(sx / (Math.Sqrt((length + s1) * (length + s2)))));
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double sx = 1.0;
+ double s1 = 1.0;
+ double s2 = 1.0;
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var col = columnIndices[c];
+ sx += x[i, col] * xt[j, col];
+ s1 += x[i, col] * x[i, col];
+ s2 += xt[j, col] * xt[j, col];
+ }
+
+ return (scale * Math.Asin(sx / (Math.Sqrt((length + s1) * (length + s2)))));
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, length, scale, columnIndices, fixedLength, fixedScale);
+ return cov;
+ }
+
+ // order of returned gradients must match the order in GetParameterValues!
+ private static IList GetGradient(double[,] x, int i, int j, double length, double scale, int[] columnIndices,
+ bool fixedLength, bool fixedScale) {
+ double sx = 1.0;
+ double s1 = 1.0;
+ double s2 = 1.0;
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var col = columnIndices[c];
+ sx += x[i, col] * x[j, col];
+ s1 += x[i, col] * x[i, col];
+ s2 += x[j, col] * x[j, col];
+ }
+ var h = (length + s1) * (length + s2);
+ var f = sx / Math.Sqrt(h);
+
+ var g = new List(2);
+ if (!fixedLength) g.Add(-scale / Math.Sqrt(1.0 - f * f) * ((length * sx * (2.0 * length + s1 + s2)) / Math.Pow(h, 3.0 / 2.0)));
+ if (!fixedScale) g.Add(2.0 * scale * Math.Asin(f));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceNoise.cs (revision 15428)
@@ -0,0 +1,99 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceNoise",
+ Description = "Noise covariance function for Gaussian processes.")]
+ public sealed class CovarianceNoise : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceNoise(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceNoise(CovarianceNoise original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceNoise()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale of noise."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceNoise(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return HasFixedScaleParameter ? 0 : 1;
+ }
+
+ public void SetParameter(double[] p) {
+ double scale;
+ GetParameterValues(p, out scale);
+ ScaleParameter.Value = new DoubleValue(scale);
+ }
+
+ private void GetParameterValues(double[] p, out double scale) {
+ int c = 0;
+ // gather parameter values
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceNoise", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double scale;
+ GetParameterValues(p, out scale);
+ var fixedScale = HasFixedScaleParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => i == j ? scale : 0.0;
+ cov.CrossCovariance = (x, xt, i, j) => Util.SqrDist(x, i, xt, j, columnIndices, 1.0) < 1e-9 ? scale : 0.0;
+ if (fixedScale)
+ cov.CovarianceGradient = (x, i, j) => new double[0];
+ else
+ cov.CovarianceGradient = (x, i, j) => new double[1] { i == j ? 2.0 * scale : 0.0 };
+ return cov;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePeriodic.cs (revision 15428)
@@ -0,0 +1,167 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovariancePeriodic", Description = "Periodic covariance function for Gaussian processes.")]
+ public sealed class CovariancePeriodic : ParameterizedNamedItem, ICovarianceFunction {
+
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter InverseLengthParameter {
+ get { return (IValueParameter)Parameters["InverseLength"]; }
+ }
+
+ public IValueParameter PeriodParameter {
+ get { return (IValueParameter)Parameters["Period"]; }
+ }
+
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ private bool HasFixedInverseLengthParameter {
+ get { return InverseLengthParameter.Value != null; }
+ }
+ private bool HasFixedPeriodParameter {
+ get { return PeriodParameter.Value != null; }
+ }
+
+
+ [StorableConstructor]
+ private CovariancePeriodic(bool deserializing) : base(deserializing) { }
+ private CovariancePeriodic(CovariancePeriodic original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovariancePeriodic()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale of the periodic covariance function."));
+ Parameters.Add(new OptionalValueParameter("InverseLength", "The inverse length parameter for the periodic covariance function."));
+ Parameters.Add(new OptionalValueParameter("Period", "The period parameter for the periodic covariance function."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovariancePeriodic(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return (HasFixedScaleParameter ? 0 : 1) +
+ (HasFixedPeriodParameter ? 0 : 1) +
+ (HasFixedInverseLengthParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double scale, inverseLength, period;
+ GetParameterValues(p, out scale, out period, out inverseLength);
+ ScaleParameter.Value = new DoubleValue(scale);
+ PeriodParameter.Value = new DoubleValue(period);
+ InverseLengthParameter.Value = new DoubleValue(inverseLength);
+ }
+
+
+ private void GetParameterValues(double[]
+ p, out double scale, out double period, out double inverseLength) {
+ // gather parameter values
+ int c = 0;
+ if (HasFixedInverseLengthParameter) {
+ inverseLength = InverseLengthParameter.Value.Value;
+ } else {
+ inverseLength = 1.0 / Math.Exp(p[c]);
+ c++;
+ }
+ if (HasFixedPeriodParameter) {
+ period = PeriodParameter.Value.Value;
+ } else {
+ period = Math.Exp(p[c]);
+ c++;
+ }
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePeriodic", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double inverseLength, period, scale;
+ GetParameterValues(p, out scale, out period, out inverseLength);
+ var fixedInverseLength = HasFixedInverseLengthParameter;
+ var fixedPeriod = HasFixedPeriodParameter;
+ var fixedScale = HasFixedScaleParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double k = i == j ? 0.0 : GetDistance(x, x, i, j, columnIndices);
+ k = Math.PI * k / period;
+ k = Math.Sin(k) * inverseLength;
+ k = k * k;
+
+ return scale * Math.Exp(-2.0 * k);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double k = GetDistance(x, xt, i, j, columnIndices);
+ k = Math.PI * k / period;
+ k = Math.Sin(k) * inverseLength;
+ k = k * k;
+
+ return scale * Math.Exp(-2.0 * k);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, period, inverseLength, fixedInverseLength, fixedPeriod, fixedScale);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double period, double inverseLength,
+ bool fixedInverseLength, bool fixedPeriod, bool fixedScale) {
+ double k = i == j ? 0.0 : Math.PI * GetDistance(x, x, i, j, columnIndices) / period;
+ double gradient = Math.Sin(k) * inverseLength;
+ gradient *= gradient;
+ var g = new List(3);
+ if (!fixedInverseLength)
+ g.Add(4.0 * scale * Math.Exp(-2.0 * gradient) * gradient);
+ if (!fixedPeriod) {
+ double r = Math.Sin(k) * inverseLength;
+ g.Add(2.0 * k * scale * Math.Exp(-2 * r * r) * Math.Sin(2 * k) * inverseLength * inverseLength);
+ }
+ if (!fixedScale)
+ g.Add(2.0 * scale * Math.Exp(-2 * gradient));
+ return g;
+ }
+
+ private static double GetDistance(double[,] x, double[,] xt, int i, int j, int[] columnIndices) {
+ return Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1));
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePiecewisePolynomial.cs (revision 15428)
@@ -0,0 +1,170 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovariancePiecewisePolynomial",
+ Description = "Piecewise polynomial covariance function with compact support for Gaussian processes.")]
+ public sealed class CovariancePiecewisePolynomial : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter LengthParameter {
+ get { return (IValueParameter)Parameters["Length"]; }
+ }
+
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IConstrainedValueParameter VParameter {
+ get { return (IConstrainedValueParameter)Parameters["V"]; }
+ }
+ private bool HasFixedLengthParameter {
+ get { return LengthParameter.Value != null; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovariancePiecewisePolynomial(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovariancePiecewisePolynomial(CovariancePiecewisePolynomial original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovariancePiecewisePolynomial()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Length", "The length parameter of the isometric piecewise polynomial covariance function."));
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter of the piecewise polynomial covariance function."));
+
+ var validValues = new ItemSet(new IntValue[] {
+ (IntValue)(new IntValue().AsReadOnly()),
+ (IntValue)(new IntValue(1).AsReadOnly()),
+ (IntValue)(new IntValue(2).AsReadOnly()),
+ (IntValue)(new IntValue(3).AsReadOnly()) });
+ Parameters.Add(new ConstrainedValueParameter("V", "The v parameter of the piecewise polynomial function (allowed values 0, 1, 2, 3).", validValues, validValues.First()));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovariancePiecewisePolynomial(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return
+ (HasFixedLengthParameter ? 0 : 1) +
+ (HasFixedScaleParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double @const, scale;
+ GetParameterValues(p, out @const, out scale);
+ LengthParameter.Value = new DoubleValue(@const);
+ ScaleParameter.Value = new DoubleValue(scale);
+ }
+
+ private void GetParameterValues(double[] p, out double length, out double scale) {
+ // gather parameter values
+ int n = 0;
+ if (HasFixedLengthParameter) {
+ length = LengthParameter.Value.Value;
+ } else {
+ length = Math.Exp(p[n]);
+ n++;
+ }
+
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[n]);
+ n++;
+ }
+ if (p.Length != n) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePiecewisePolynomial", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double length, scale;
+ int v = VParameter.Value.Value;
+ GetParameterValues(p, out length, out scale);
+ var fixedLength = HasFixedLengthParameter;
+ var fixedScale = HasFixedScaleParameter;
+ int exp = (int)Math.Floor(columnIndices.Count() / 2.0) + v + 1;
+
+ Func f;
+ Func df;
+ switch (v) {
+ case 0:
+ f = (r) => 1.0;
+ df = (r) => 0.0;
+ break;
+ case 1:
+ f = (r) => 1 + (exp + 1) * r;
+ df = (r) => exp + 1;
+ break;
+ case 2:
+ f = (r) => 1 + (exp + 2) * r + (exp * exp + 4.0 * exp + 3) / 3.0 * r * r;
+ df = (r) => (exp + 2) + 2 * (exp * exp + 4.0 * exp + 3) / 3.0 * r;
+ break;
+ case 3:
+ f = (r) => 1 + (exp + 3) * r + (6.0 * exp * exp + 36 * exp + 45) / 15.0 * r * r +
+ (exp * exp * exp + 9 * exp * exp + 23 * exp + 45) / 15.0 * r * r * r;
+ df = (r) => (exp + 3) + 2 * (6.0 * exp * exp + 36 * exp + 45) / 15.0 * r +
+ (exp * exp * exp + 9 * exp * exp + 23 * exp + 45) / 5.0 * r * r;
+ break;
+ default: throw new ArgumentException();
+ }
+
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
+ return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double k = Math.Sqrt(Util.SqrDist(x, i, xt, j, columnIndices, 1.0 / length));
+ return scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, length, scale, v, exp, f, df, columnIndices, fixedLength, fixedScale);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, double length, double scale, int v, double exp, Func f, Func df, int[] columnIndices,
+ bool fixedLength, bool fixedScale) {
+ double k = Math.Sqrt(Util.SqrDist(x, i, x, j, columnIndices, 1.0 / length));
+ var g = new List(2);
+ if (!fixedLength) g.Add(scale * Math.Pow(Math.Max(1.0 - k, 0), exp + v - 1) * k * ((exp + v) * f(k) - Math.Max(1 - k, 0) * df(k)));
+ if (!fixedScale) g.Add(2.0 * scale * Math.Pow(Math.Max(1 - k, 0), exp + v) * f(k));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovariancePolynomial.cs (revision 15428)
@@ -0,0 +1,132 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovariancePolynomial",
+ Description = "Polynomial covariance function for Gaussian processes.")]
+ public sealed class CovariancePolynomial : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ConstParameter {
+ get { return (IValueParameter)Parameters["Const"]; }
+ }
+
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter DegreeParameter {
+ get { return (IValueParameter)Parameters["Degree"]; }
+ }
+ private bool HasFixedConstParameter {
+ get { return ConstParameter.Value != null; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovariancePolynomial(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovariancePolynomial(CovariancePolynomial original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovariancePolynomial()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Const", "Additive constant in the polymomial."));
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter of the polynomial covariance function."));
+ Parameters.Add(new ValueParameter("Degree", "The degree of the polynomial (only non-zero positive values allowed).", new IntValue(2)));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovariancePolynomial(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return
+ (HasFixedConstParameter ? 0 : 1) +
+ (HasFixedScaleParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double @const, scale;
+ GetParameterValues(p, out @const, out scale);
+ ConstParameter.Value = new DoubleValue(@const);
+ ScaleParameter.Value = new DoubleValue(scale);
+ }
+
+ private void GetParameterValues(double[] p, out double @const, out double scale) {
+ // gather parameter values
+ int n = 0;
+ if (HasFixedConstParameter) {
+ @const = ConstParameter.Value.Value;
+ } else {
+ @const = Math.Exp(p[n]);
+ n++;
+ }
+
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[n]);
+ n++;
+ }
+ if (p.Length != n) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovariancePolynomial", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double @const, scale;
+ int degree = DegreeParameter.Value.Value;
+ if (degree <= 0) throw new ArgumentException("The degree parameter for CovariancePolynomial must be greater than zero.");
+ GetParameterValues(p, out @const, out scale);
+ var fixedConst = HasFixedConstParameter;
+ var fixedScale = HasFixedScaleParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, j, columnIndices, 1.0), degree);
+ cov.CrossCovariance = (x, xt, i, j) => scale * Math.Pow(@const + Util.ScalarProd(x, i, xt, j, columnIndices, 1.0), degree);
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, @const, scale, degree, columnIndices, fixedConst, fixedScale);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, double c, double scale, int degree, int[] columnIndices,
+ bool fixedConst, bool fixedScale) {
+ double s = Util.ScalarProd(x, i, j, columnIndices, 1.0);
+ var g = new List(2);
+ if (!fixedConst) g.Add(c * degree * scale * Math.Pow(c + s, degree - 1));
+ if (!fixedScale) g.Add(2 * scale * Math.Pow(c + s, degree));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceProduct.cs (revision 15428)
@@ -0,0 +1,108 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceProduct",
+ Description = "Product covariance function for Gaussian processes.")]
+ public sealed class CovarianceProduct : Item, ICovarianceFunction {
+ [Storable]
+ private ItemList factors;
+
+ [Storable]
+ private int numberOfVariables;
+ public ItemList Factors {
+ get { return factors; }
+ }
+
+ [StorableConstructor]
+ private CovarianceProduct(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceProduct(CovarianceProduct original, Cloner cloner)
+ : base(original, cloner) {
+ this.factors = cloner.Clone(original.factors);
+ this.numberOfVariables = original.numberOfVariables;
+ }
+
+ public CovarianceProduct()
+ : base() {
+ this.factors = new ItemList();
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceProduct(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ this.numberOfVariables = numberOfVariables;
+ return factors.Select(f => f.GetNumberOfParameters(numberOfVariables)).Sum();
+ }
+
+ public void SetParameter(double[] p) {
+ int offset = 0;
+ foreach (var f in factors) {
+ var numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+ f.SetParameter(p.Skip(offset).Take(numberOfParameters).ToArray());
+ offset += numberOfParameters;
+ }
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ if (factors.Count == 0) throw new ArgumentException("at least one factor is necessary for the product covariance function.");
+ var functions = new List();
+ foreach (var f in factors) {
+ int numberOfParameters = f.GetNumberOfParameters(numberOfVariables);
+ functions.Add(f.GetParameterizedCovarianceFunction(p.Take(numberOfParameters).ToArray(), columnIndices));
+ p = p.Skip(numberOfParameters).ToArray();
+ }
+
+
+ var product = new ParameterizedCovarianceFunction();
+ product.Covariance = (x, i, j) => functions.Select(e => e.Covariance(x, i, j)).Aggregate((a, b) => a * b);
+ product.CrossCovariance = (x, xt, i, j) => functions.Select(e => e.CrossCovariance(x, xt, i, j)).Aggregate((a, b) => a * b);
+ product.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, functions);
+ return product;
+ }
+
+ public static IList GetGradient(double[,] x, int i, int j, List factorFunctions) {
+ var covariances = factorFunctions.Select(f => f.Covariance(x, i, j)).ToArray();
+ var gr = new List();
+ for (int ii = 0; ii < factorFunctions.Count; ii++) {
+ foreach (var g in factorFunctions[ii].CovarianceGradient(x, i, j)) {
+ double res = g;
+ for (int jj = 0; jj < covariances.Length; jj++)
+ if (ii != jj) res *= covariances[jj];
+ gr.Add(res);
+ }
+ }
+ return gr;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticArd.cs (revision 15428)
@@ -0,0 +1,167 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceRationalQuadraticArd",
+ Description = "Rational quadratic covariance function with automatic relevance determination for Gaussian processes.")]
+ public sealed class CovarianceRationalQuadraticArd : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter InverseLengthParameter {
+ get { return (IValueParameter)Parameters["InverseLength"]; }
+ }
+
+ public IValueParameter ShapeParameter {
+ get { return (IValueParameter)Parameters["Shape"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ private bool HasFixedInverseLengthParameter {
+ get { return InverseLengthParameter.Value != null; }
+ }
+ private bool HasFixedShapeParameter {
+ get { return ShapeParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceRationalQuadraticArd(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceRationalQuadraticArd(CovarianceRationalQuadraticArd original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceRationalQuadraticArd()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter of the rational quadratic covariance function with ARD."));
+ Parameters.Add(new OptionalValueParameter("InverseLength", "The inverse length parameter for automatic relevance determination."));
+ Parameters.Add(new OptionalValueParameter("Shape", "The shape parameter (alpha) of the rational quadratic covariance function with ARD."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceRationalQuadraticArd(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return
+ (HasFixedScaleParameter ? 0 : 1) +
+ (HasFixedShapeParameter ? 0 : 1) +
+ (HasFixedInverseLengthParameter ? 0 : numberOfVariables);
+ }
+
+ public void SetParameter(double[] p) {
+ double scale, shape;
+ double[] inverseLength;
+ GetParameterValues(p, out scale, out shape, out inverseLength);
+ ScaleParameter.Value = new DoubleValue(scale);
+ ShapeParameter.Value = new DoubleValue(shape);
+ InverseLengthParameter.Value = new DoubleArray(inverseLength);
+ }
+
+ private void GetParameterValues(double[] p, out double scale, out double shape, out double[] inverseLength) {
+ int c = 0;
+ // gather parameter values
+ if (HasFixedInverseLengthParameter) {
+ inverseLength = InverseLengthParameter.Value.ToArray();
+ } else {
+ int length = p.Length;
+ if (!HasFixedScaleParameter) length--;
+ if (!HasFixedShapeParameter) length--;
+ inverseLength = p.Select(e => 1.0 / Math.Exp(e)).Take(length).ToArray();
+ c += inverseLength.Length;
+ }
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (HasFixedShapeParameter) {
+ shape = ShapeParameter.Value.Value;
+ } else {
+ shape = Math.Exp(p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticArd", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double scale, shape;
+ double[] inverseLength;
+ GetParameterValues(p, out scale, out shape, out inverseLength);
+ var fixedInverseLength = HasFixedInverseLengthParameter;
+ var fixedScale = HasFixedScaleParameter;
+ var fixedShape = HasFixedShapeParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double d = i == j
+ ? 0.0
+ : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+ return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double d = Util.SqrDist(x, i, xt, j, inverseLength, columnIndices);
+ return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength, fixedInverseLength, fixedScale, fixedShape);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double[] inverseLength,
+ bool fixedInverseLength, bool fixedScale, bool fixedShape) {
+ double d = i == j
+ ? 0.0
+ : Util.SqrDist(x, i, j, inverseLength, columnIndices);
+ double b = 1 + 0.5 * d / shape;
+ int k = 0;
+ var g = new List(columnIndices.Length + 2);
+ if (!fixedInverseLength) {
+ foreach (var columnIndex in columnIndices) {
+ g.Add(
+ scale * Math.Pow(b, -shape - 1) *
+ Util.SqrDist(x[i, columnIndex] * inverseLength[k], x[j, columnIndex] * inverseLength[k]));
+ k++;
+ }
+ }
+ if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+ if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceRationalQuadraticIso.cs (revision 15428)
@@ -0,0 +1,155 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceRationalQuadraticIso",
+ Description = "Isotropic rational quadratic covariance function for Gaussian processes.")]
+ public sealed class CovarianceRationalQuadraticIso : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter InverseLengthParameter {
+ get { return (IValueParameter)Parameters["InverseLength"]; }
+ }
+
+ public IValueParameter ShapeParameter {
+ get { return (IValueParameter)Parameters["Shape"]; }
+ }
+
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+ private bool HasFixedInverseLengthParameter {
+ get { return InverseLengthParameter.Value != null; }
+ }
+ private bool HasFixedShapeParameter {
+ get { return ShapeParameter.Value != null; }
+ }
+
+
+ [StorableConstructor]
+ private CovarianceRationalQuadraticIso(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceRationalQuadraticIso(CovarianceRationalQuadraticIso original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceRationalQuadraticIso()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter of the isometric rational quadratic covariance function."));
+ Parameters.Add(new OptionalValueParameter("InverseLength", "The inverse length parameter of the isometric rational quadratic covariance function."));
+ Parameters.Add(new OptionalValueParameter("Shape", "The shape parameter (alpha) of the isometric rational quadratic covariance function."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceRationalQuadraticIso(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return (HasFixedScaleParameter ? 0 : 1) +
+ (HasFixedShapeParameter ? 0 : 1) +
+ (HasFixedInverseLengthParameter ? 0 : 1);
+ }
+
+ public void SetParameter(double[] p) {
+ double scale, shape, inverseLength;
+ GetParameterValues(p, out scale, out shape, out inverseLength);
+ ScaleParameter.Value = new DoubleValue(scale);
+ ShapeParameter.Value = new DoubleValue(shape);
+ InverseLengthParameter.Value = new DoubleValue(inverseLength);
+ }
+
+ private void GetParameterValues(double[] p, out double scale, out double shape, out double inverseLength) {
+ int c = 0;
+ // gather parameter values
+ if (HasFixedInverseLengthParameter) {
+ inverseLength = InverseLengthParameter.Value.Value;
+ } else {
+ inverseLength = 1.0 / Math.Exp(p[c]);
+ c++;
+ }
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[c]);
+ c++;
+ }
+ if (HasFixedShapeParameter) {
+ shape = ShapeParameter.Value.Value;
+ } else {
+ shape = Math.Exp(p[c]);
+ c++;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceRationalQuadraticIso", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double scale, shape, inverseLength;
+ GetParameterValues(p, out scale, out shape, out inverseLength);
+ var fixedInverseLength = HasFixedInverseLengthParameter;
+ var fixedScale = HasFixedScaleParameter;
+ var fixedShape = HasFixedShapeParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ double d = i == j
+ ? 0.0
+ : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+ return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ double d = Util.SqrDist(x, i, xt, j, columnIndices, inverseLength);
+ return scale * Math.Pow(1 + 0.5 * d / shape, -shape);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, shape, inverseLength, fixedInverseLength, fixedScale, fixedShape);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, double shape, double inverseLength,
+ bool fixedInverseLength, bool fixedScale, bool fixedShape) {
+ double d = i == j
+ ? 0.0
+ : Util.SqrDist(x, i, j, columnIndices, inverseLength);
+
+ double b = 1 + 0.5 * d / shape;
+ var g = new List(3);
+ if (!fixedInverseLength) g.Add(scale * Math.Pow(b, -shape - 1) * d);
+ if (!fixedScale) g.Add(2 * scale * Math.Pow(b, -shape));
+ if (!fixedShape) g.Add(scale * Math.Pow(b, -shape) * (0.5 * d / b - shape * Math.Log(b)));
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceScale.cs (revision 15428)
@@ -0,0 +1,113 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceScale",
+ Description = "Scale covariance function for Gaussian processes.")]
+ public sealed class CovarianceScale : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+ private bool HasFixedScaleParameter {
+ get { return ScaleParameter.Value != null; }
+ }
+
+ public IValueParameter CovarianceFunctionParameter {
+ get { return (IValueParameter)Parameters["CovarianceFunction"]; }
+ }
+
+ [StorableConstructor]
+ private CovarianceScale(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceScale(CovarianceScale original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceScale()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+
+ Parameters.Add(new OptionalValueParameter("Scale", "The scale parameter."));
+ Parameters.Add(new ValueParameter("CovarianceFunction", "The covariance function that should be scaled.", new CovarianceSquaredExponentialIso()));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceScale(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ return (HasFixedScaleParameter ? 0 : 1) + CovarianceFunctionParameter.Value.GetNumberOfParameters(numberOfVariables);
+ }
+
+ public void SetParameter(double[] p) {
+ double scale;
+ GetParameterValues(p, out scale);
+ ScaleParameter.Value = new DoubleValue(scale);
+ CovarianceFunctionParameter.Value.SetParameter(p.Skip(1).ToArray());
+ }
+
+ private void GetParameterValues(double[] p, out double scale) {
+ // gather parameter values
+ if (HasFixedScaleParameter) {
+ scale = ScaleParameter.Value.Value;
+ } else {
+ scale = Math.Exp(2 * p[0]);
+ }
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double scale;
+ GetParameterValues(p, out scale);
+ var fixedScale = HasFixedScaleParameter;
+ var subCov = CovarianceFunctionParameter.Value.GetParameterizedCovarianceFunction(p.Skip(1).ToArray(), columnIndices);
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => scale * subCov.Covariance(x, i, j);
+ cov.CrossCovariance = (x, xt, i, j) => scale * subCov.CrossCovariance(x, xt, i, j);
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, columnIndices, scale, subCov, fixedScale);
+ return cov;
+ }
+
+ private static IList GetGradient(double[,] x, int i, int j, int[] columnIndices, double scale, ParameterizedCovarianceFunction cov,
+ bool fixedScale) {
+ var gr = new List((!fixedScale ? 1 : 0) + cov.CovarianceGradient(x, i, j).Count);
+ if (!fixedScale) {
+ gr.Add(2 * scale * cov.Covariance(x, i, j));
+ }
+ foreach (var g in cov.CovarianceGradient(x, i, j))
+ gr.Add(scale * g);
+ return gr;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSpectralMixture.cs (revision 15428)
@@ -0,0 +1,240 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceSpectralMixture",
+ Description = "The spectral mixture kernel described in Wilson A. G. and Adams R.P., Gaussian Process Kernels for Pattern Discovery and Exptrapolation, ICML 2013.")]
+ public sealed class CovarianceSpectralMixture : ParameterizedNamedItem, ICovarianceFunction {
+ public const string QParameterName = "Number of components (Q)";
+ public const string WeightParameterName = "Weight";
+ public const string FrequencyParameterName = "Component frequency (mu)";
+ public const string LengthScaleParameterName = "Length scale (nu)";
+ public IValueParameter QParameter {
+ get { return (IValueParameter)Parameters[QParameterName]; }
+ }
+
+ public IValueParameter WeightParameter {
+ get { return (IValueParameter)Parameters[WeightParameterName]; }
+ }
+ public IValueParameter FrequencyParameter {
+ get { return (IValueParameter)Parameters[FrequencyParameterName]; }
+ }
+
+ public IValueParameter LengthScaleParameter {
+ get { return (IValueParameter)Parameters[LengthScaleParameterName]; }
+ }
+
+ private bool HasFixedWeightParameter {
+ get { return WeightParameter.Value != null; }
+ }
+ private bool HasFixedFrequencyParameter {
+ get { return FrequencyParameter.Value != null; }
+ }
+ private bool HasFixedLengthScaleParameter {
+ get { return LengthScaleParameter.Value != null; }
+ }
+
+ [StorableConstructor]
+ private CovarianceSpectralMixture(bool deserializing)
+ : base(deserializing) {
+ }
+
+ private CovarianceSpectralMixture(CovarianceSpectralMixture original, Cloner cloner)
+ : base(original, cloner) {
+ }
+
+ public CovarianceSpectralMixture()
+ : base() {
+ Name = ItemName;
+ Description = ItemDescription;
+ Parameters.Add(new ValueParameter(QParameterName, "The number of Gaussians (Q) to use for the spectral mixture.", new IntValue(10)));
+ Parameters.Add(new OptionalValueParameter(WeightParameterName, "The weight of the component w (peak height of the Gaussian in spectrum)."));
+ Parameters.Add(new OptionalValueParameter(FrequencyParameterName, "The inverse component period parameter mu_q (location of the Gaussian in spectrum)."));
+ Parameters.Add(new OptionalValueParameter(LengthScaleParameterName, "The length scale parameter (nu_q) (variance of the Gaussian in the spectrum)."));
+ }
+
+ public override IDeepCloneable Clone(Cloner cloner) {
+ return new CovarianceSpectralMixture(this, cloner);
+ }
+
+ public int GetNumberOfParameters(int numberOfVariables) {
+ var q = QParameter.Value.Value;
+ return
+ (HasFixedWeightParameter ? 0 : q) +
+ (HasFixedFrequencyParameter ? 0 : q * numberOfVariables) +
+ (HasFixedLengthScaleParameter ? 0 : q * numberOfVariables);
+ }
+
+ public void SetParameter(double[] p) {
+ double[] weight, frequency, lengthScale;
+ GetParameterValues(p, out weight, out frequency, out lengthScale);
+ WeightParameter.Value = new DoubleArray(weight);
+ FrequencyParameter.Value = new DoubleArray(frequency);
+ LengthScaleParameter.Value = new DoubleArray(lengthScale);
+ }
+
+
+ private void GetParameterValues(double[] p, out double[] weight, out double[] frequency, out double[] lengthScale) {
+ // gather parameter values
+ int c = 0;
+ int q = QParameter.Value.Value;
+ // guess number of elements for frequency and length (=q * numberOfVariables)
+ int n = WeightParameter.Value == null ? ((p.Length - q) / 2) : (p.Length / 2);
+ if (HasFixedWeightParameter) {
+ weight = WeightParameter.Value.ToArray();
+ } else {
+ weight = p.Skip(c).Select(Math.Exp).Take(q).ToArray();
+ c += q;
+ }
+ if (HasFixedFrequencyParameter) {
+ frequency = FrequencyParameter.Value.ToArray();
+ } else {
+ frequency = p.Skip(c).Select(Math.Exp).Take(n).ToArray();
+ c += n;
+ }
+ if (HasFixedLengthScaleParameter) {
+ lengthScale = LengthScaleParameter.Value.ToArray();
+ } else {
+ lengthScale = p.Skip(c).Select(Math.Exp).Take(n).ToArray();
+ c += n;
+ }
+ if (p.Length != c) throw new ArgumentException("The length of the parameter vector does not match the number of free parameters for CovarianceSpectralMixture", "p");
+ }
+
+ public ParameterizedCovarianceFunction GetParameterizedCovarianceFunction(double[] p, int[] columnIndices) {
+ double[] weight, frequency, lengthScale;
+ GetParameterValues(p, out weight, out frequency, out lengthScale);
+ var fixedWeight = HasFixedWeightParameter;
+ var fixedFrequency = HasFixedFrequencyParameter;
+ var fixedLengthScale = HasFixedLengthScaleParameter;
+ // create functions
+ var cov = new ParameterizedCovarianceFunction();
+ cov.Covariance = (x, i, j) => {
+ return GetCovariance(x, x, i, j, QParameter.Value.Value, weight, frequency,
+ lengthScale, columnIndices);
+ };
+ cov.CrossCovariance = (x, xt, i, j) => {
+ return GetCovariance(x, xt, i, j, QParameter.Value.Value, weight, frequency,
+ lengthScale, columnIndices);
+ };
+ cov.CovarianceGradient = (x, i, j) => GetGradient(x, i, j, QParameter.Value.Value, weight, frequency,
+ lengthScale, columnIndices, fixedWeight, fixedFrequency, fixedLengthScale);
+ return cov;
+ }
+
+ private static double GetCovariance(double[,] x, double[,] xt, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices) {
+ // tau = x - x' (only for selected variables)
+ double[] tau =
+ Util.GetRow(x, i, columnIndices).Zip(Util.GetRow(xt, j, columnIndices), (xi, xj) => xi - xj).ToArray();
+ int numberOfVariables = lengthScale.Length / maxQ;
+ double k = 0;
+ // for each component
+ for (int q = 0; q < maxQ; q++) {
+ double kc = weight[q]; // weighted kernel component
+
+ int idx = 0; // helper index for tau
+ // for each selected variable
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var col = columnIndices[c];
+ kc *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
+ idx++;
+ }
+ k += kc;
+ }
+ return k;
+ }
+
+ public static double f1(double tau, double lengthScale) {
+ return Math.Exp(-2 * Math.PI * Math.PI * tau * tau * lengthScale);
+ }
+ public static double f2(double tau, double frequency) {
+ return Math.Cos(2 * Math.PI * tau * frequency);
+ }
+
+ // order of returned gradients must match the order in GetParameterValues!
+ private static IList GetGradient(double[,] x, int i, int j, int maxQ, double[] weight, double[] frequency, double[] lengthScale, int[] columnIndices,
+ bool fixedWeight, bool fixedFrequency, bool fixedLengthScale) {
+ double[] tau = Util.GetRow(x, i, columnIndices).Zip(Util.GetRow(x, j, columnIndices), (xi, xj) => xi - xj).ToArray();
+ int numberOfVariables = lengthScale.Length / maxQ;
+
+ var g = new List((!fixedWeight ? maxQ : 0) + (!fixedFrequency ? maxQ * columnIndices.Length : 0) + (!fixedLengthScale ? maxQ * columnIndices.Length : 0));
+ if (!fixedWeight) {
+ // weight
+ // for each component
+ for (int q = 0; q < maxQ; q++) {
+ double k = weight[q];
+ int idx = 0; // helper index for tau
+ // for each selected variable
+ for (int c = 0; c < columnIndices.Length; c++) {
+ var col = columnIndices[c];
+ k *= f1(tau[idx], lengthScale[q * numberOfVariables + col]) * f2(tau[idx], frequency[q * numberOfVariables + col]);
+ idx++;
+ }
+ g.Add(k);
+ }
+ }
+
+ if (!fixedFrequency) {
+ // frequency
+ // for each component
+ for (int q = 0; q < maxQ; q++) {
+ int idx = 0; // helper index for tau
+ // for each selected variable
+ foreach (var c in columnIndices) {
+ double k = f1(tau[idx], lengthScale[q * numberOfVariables + c]) *
+ -2 * Math.PI * tau[idx] * frequency[q * numberOfVariables + c] *
+ Math.Sin(2 * Math.PI * tau[idx] * frequency[q * numberOfVariables + c]);
+ idx++;
+ g.Add(weight[q] * k);
+ }
+ }
+ }
+
+ if (!fixedLengthScale) {
+ // length scale
+ // for each component
+ for (int q = 0; q < maxQ; q++) {
+ int idx = 0; // helper index for tau
+ // for each selected variable
+ foreach (var c in columnIndices) {
+ double k = -2 * Math.PI * Math.PI * tau[idx] * tau[idx] * lengthScale[q * numberOfVariables + c] *
+ f1(tau[idx], lengthScale[q * numberOfVariables + c]) *
+ f2(tau[idx], frequency[q * numberOfVariables + c]);
+ idx++;
+ g.Add(weight[q] * k);
+ }
+ }
+ }
+
+ return g;
+ }
+ }
+}
Index: /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs
===================================================================
--- /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs (revision 15428)
+++ /branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/CovarianceFunctions/CovarianceSquaredExponentialArd.cs (revision 15428)
@@ -0,0 +1,144 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see .
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+ [StorableClass]
+ [Item(Name = "CovarianceSquaredExponentialArd", Description = "Squared exponential covariance function with automatic relevance determination for Gaussian processes.")]
+ public sealed class CovarianceSquaredExponentialArd : ParameterizedNamedItem, ICovarianceFunction {
+ public IValueParameter ScaleParameter {
+ get { return (IValueParameter)Parameters["Scale"]; }
+ }
+
+ public IValueParameter