Free cookie consent management tool by TermsFeed Policy Generator

Changeset 8396 for trunk/sources


Ignore:
Timestamp:
08/02/12 17:27:46 (12 years ago)
Author:
gkronber
Message:

#1902 implemented LM-BFGS algorithm and improved GPR

Location:
trunk/sources
Files:
1 added
7 edited
6 moved

Legend:

Unmodified
Added
Removed

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessHyperparameterInitializer.cs

    r8375 r8396  
    2424using HeuristicLab.Core;
    2525using HeuristicLab.Data;
     26using HeuristicLab.Encodings.RealVectorEncoding;
    2627using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2728using HeuristicLab.Operators;
    … …  
    3233namespace HeuristicLab.Algorithms.DataAnalysis {
    3334  [StorableClass]
    34   [Item(Name = "GaussianProcessSetHyperparameterLength",
    35     Description = "Determines the length of the hyperparameter vector based on the mean function, covariance function, and number of allowed input variables.")]
    36   public sealed class GaussianProcessSetHyperparameterLength : SingleSuccessorOperator {
     35  [Item(Name = "GaussianProcessHyperparameterInitializer",
     36    Description = "Initializers the hyperparameter vector based on the mean function, covariance function, and number of allowed input variables.")]
     37  public sealed class GaussianProcessHyperparameterInitializer : SingleSuccessorOperator {
    3738    private const string MeanFunctionParameterName = "MeanFunction";
    3839    private const string CovarianceFunctionParameterName = "CovarianceFunction";
    3940    private const string ProblemDataParameterName = "ProblemData";
    40     private const string NumberOfHyperparameterParameterName = "NumberOfHyperparameter";
     41    private const string HyperparameterParameterName = "Hyperparameter";
    4142
    4243    #region Parameter Properties
    … …  
    5253    }
    5354    // out
    54     public ILookupParameter<IntValue> NumberOfHyperparameterParameter {
    55       get { return (ILookupParameter<IntValue>)Parameters[NumberOfHyperparameterParameterName]; }
     55    public ILookupParameter<RealVector> HyperparameterParameter {
     56      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
    5657    }
    5758    #endregion
    … …  
    6465
    6566    [StorableConstructor]
    66     private GaussianProcessSetHyperparameterLength(bool deserializing) : base(deserializing) { }
    67     private GaussianProcessSetHyperparameterLength(GaussianProcessSetHyperparameterLength original, Cloner cloner) : base(original, cloner) { }
    68     public GaussianProcessSetHyperparameterLength()
     67    private GaussianProcessHyperparameterInitializer(bool deserializing) : base(deserializing) { }
     68    private GaussianProcessHyperparameterInitializer(GaussianProcessHyperparameterInitializer original, Cloner cloner) : base(original, cloner) { }
     69    public GaussianProcessHyperparameterInitializer()
    6970      : base() {
    7071      // in
    … …  
    7374      Parameters.Add(new LookupParameter<IDataAnalysisProblemData>(ProblemDataParameterName, "The input data for the Gaussian process."));
    7475      // out
    75       Parameters.Add(new LookupParameter<IntValue>(NumberOfHyperparameterParameterName, "The length of the hyperparameter vector for the Gaussian process model."));
     76      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The initial hyperparameter vector for the Gaussian process model."));
    7677    }
    7778
    7879    public override IDeepCloneable Clone(Cloner cloner) {
    79       return new GaussianProcessSetHyperparameterLength(this, cloner);
     80      return new GaussianProcessHyperparameterInitializer(this, cloner);
    8081    }
    8182
    82     public override IOperation Apply()
    83     {
     83    public override IOperation Apply() {
    8484      var inputVariablesCount = ProblemData.AllowedInputVariables.Count();
    8585      int l = 1 + MeanFunction.GetNumberOfParameters(inputVariablesCount) +
    8686              CovarianceFunction.GetNumberOfParameters(inputVariablesCount);
    87       NumberOfHyperparameterParameter.ActualValue = new IntValue(l);
     87      HyperparameterParameter.ActualValue = new RealVector(l);
    8888      return base.Apply();
    8989    }

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModel.cs

    r8371 r8396  
    3535  [Item("GaussianProcessModel", "Represents a Gaussian process posterior.")]
    3636  public sealed class GaussianProcessModel : NamedItem, IGaussianProcessModel {
    37 
    3837    [Storable]
    3938    private double negativeLogLikelihood;

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessModelCreator.cs

    r8375 r8396  
    2323using HeuristicLab.Core;
    2424using HeuristicLab.Data;
     25using HeuristicLab.Encodings.RealVectorEncoding;
    2526using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2627using HeuristicLab.Operators;
    … …  
    4243    #region Parameter Properties
    4344    // in
    44     public ILookupParameter<DoubleArray> HyperparameterParameter {
    45       get { return (ILookupParameter<DoubleArray>)Parameters[HyperparameterParameterName]; }
     45    public ILookupParameter<RealVector> HyperparameterParameter {
     46      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterParameterName]; }
    4647    }
    4748    public ILookupParameter<IMeanFunction> MeanFunctionParameter {
    … …  
    5556      get { return (ILookupParameter<IGaussianProcessModel>)Parameters[ModelParameterName]; }
    5657    }
    57     public ILookupParameter<DoubleArray> HyperparameterGradientsParameter {
    58       get { return (ILookupParameter<DoubleArray>)Parameters[HyperparameterGradientsParameterName]; }
     58    public ILookupParameter<RealVector> HyperparameterGradientsParameter {
     59      get { return (ILookupParameter<RealVector>)Parameters[HyperparameterGradientsParameterName]; }
    5960    }
    6061    public ILookupParameter<DoubleValue> NegativeLogLikelihoodParameter {
    … …  
    6566
    6667    #region Properties
    67     protected DoubleArray Hyperparameter { get { return HyperparameterParameter.ActualValue; } }
     68    protected RealVector Hyperparameter { get { return HyperparameterParameter.ActualValue; } }
    6869    protected IMeanFunction MeanFunction { get { return MeanFunctionParameter.ActualValue; } }
    6970    protected ICovarianceFunction CovarianceFunction { get { return CovarianceFunctionParameter.ActualValue; } }
    … …  
    7677      : base() {
    7778      // in
    78       Parameters.Add(new LookupParameter<DoubleArray>(HyperparameterParameterName, "The hyperparameters for the Gaussian process model."));
     79      Parameters.Add(new LookupParameter<RealVector>(HyperparameterParameterName, "The hyperparameters for the Gaussian process model."));
    7980      Parameters.Add(new LookupParameter<IMeanFunction>(MeanFunctionParameterName, "The mean function for the Gaussian process model."));
    8081      Parameters.Add(new LookupParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function for the Gaussian process model."));
    8182      // out
    8283      Parameters.Add(new LookupParameter<IGaussianProcessModel>(ModelParameterName, "The resulting Gaussian process model"));
    83       Parameters.Add(new LookupParameter<DoubleArray>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
     84      Parameters.Add(new LookupParameter<RealVector>(HyperparameterGradientsParameterName, "The gradients of the hyperparameters for the produced Gaussian process model (necessary for hyperparameter optimization)"));
    8485      Parameters.Add(new LookupParameter<DoubleValue>(NegativeLogLikelihoodParameterName, "The negative log-likelihood of the produced Gaussian process model given the data."));
    8586    }

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegression.cs

    r8375 r8396  
    2828using HeuristicLab.Core;
    2929using HeuristicLab.Data;
     30using HeuristicLab.Encodings.RealVectorEncoding;
    3031using HeuristicLab.Operators;
    3132using HeuristicLab.Optimization;
    … …  
    5556    private const string CovarianceFunctionParameterName = "CovarianceFunction";
    5657    private const string MinimizationIterationsParameterName = "Iterations";
     58    private const string ApproximateGradientsParameterName = "ApproximateGradients";
    5759
    5860    #region parameter properties
    … …  
    8890    public GaussianProcessRegression()
    8991      : base() {
     92      this.name = ItemName;
     93      this.description = ItemDescription;
     94
    9095      Problem = new RegressionProblem();
    9196
    … …  
    97102      Parameters.Add(new ConstrainedValueParameter<ICovarianceFunction>(CovarianceFunctionParameterName, "The covariance function to use.",
    98103        new ItemSet<ICovarianceFunction>(covFunctions), covFunctions.First()));
    99       Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with BFGS.", new IntValue(20)));
     104      Parameters.Add(new ValueParameter<IntValue>(MinimizationIterationsParameterName, "The number of iterations for likelihood optimization with LM-BFGS.", new IntValue(20)));
     105      Parameters.Add(new ValueParameter<BoolValue>(ApproximateGradientsParameterName, "Indicates that gradients should not be approximated (necessary for LM-BFGS).", new BoolValue(false)));
     106      Parameters[ApproximateGradientsParameterName].Hidden = true; // should not be changed
    100107
    101       var setParameterLength = new GaussianProcessSetHyperparameterLength();
    102       var initializer = new BFGSInitializer();
    103       var makeStep = new BFGSMakeStep();
     108      var gpInitializer = new GaussianProcessHyperparameterInitializer();
     109      var bfgsInitializer = new LbfgsInitializer();
     110      var makeStep = new LbfgsMakeStep();
    104111      var branch = new ConditionalBranch();
    105112      var modelCreator = new GaussianProcessRegressionModelCreator();
    106       var updateResults = new BFGSUpdateResults();
    107       var analyzer = new BFGSAnalyzer();
     113      var updateResults = new LbfgsUpdateResults();
     114      var analyzer = new LbfgsAnalyzer();
    108115      var finalModelCreator = new GaussianProcessRegressionModelCreator();
    109       var finalAnalyzer = new BFGSAnalyzer();
     116      var finalAnalyzer = new LbfgsAnalyzer();
    110117      var solutionCreator = new GaussianProcessRegressionSolutionCreator();
    111118
    112       OperatorGraph.InitialOperator = setParameterLength;
     119      OperatorGraph.InitialOperator = gpInitializer;
    113120
    114       setParameterLength.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
    115       setParameterLength.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
    116       setParameterLength.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
    117       setParameterLength.Successor = initializer;
     121      gpInitializer.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
     122      gpInitializer.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
     123      gpInitializer.ProblemDataParameter.ActualName = Problem.ProblemDataParameter.Name;
     124      gpInitializer.HyperparameterParameter.ActualName = modelCreator.HyperparameterParameter.Name;
     125      gpInitializer.Successor = bfgsInitializer;
    118126
    119       initializer.IterationsParameter.ActualName = MinimizationIterationsParameterName;
    120       initializer.DimensionParameter.ActualName = setParameterLength.NumberOfHyperparameterParameter.Name;
    121       initializer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
    122       initializer.Successor = makeStep;
     127      bfgsInitializer.IterationsParameter.ActualName = MinimizationIterationsParameterName;
     128      bfgsInitializer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
     129      bfgsInitializer.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
     130      bfgsInitializer.Successor = makeStep;
    123131
    124       makeStep.BFGSStateParameter.ActualName = initializer.BFGSStateParameter.Name;
     132      makeStep.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
    125133      makeStep.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
    126134      makeStep.Successor = branch;
    … …  
    135143      modelCreator.Successor = updateResults;
    136144
    137       updateResults.BFGSStateParameter.ActualName = initializer.BFGSStateParameter.Name;
     145      updateResults.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
    138146      updateResults.QualityParameter.ActualName = modelCreator.NegativeLogLikelihoodParameter.Name;
    139147      updateResults.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
     148      updateResults.ApproximateGradientsParameter.ActualName = ApproximateGradientsParameterName;
    140149      updateResults.Successor = analyzer;
    141150
    … …  
    143152      analyzer.PointParameter.ActualName = modelCreator.HyperparameterParameter.Name;
    144153      analyzer.QualityGradientsParameter.ActualName = modelCreator.HyperparameterGradientsParameter.Name;
    145       analyzer.BFGSStateParameter.ActualName = initializer.BFGSStateParameter.Name;
     154      analyzer.StateParameter.ActualName = bfgsInitializer.StateParameter.Name;
    146155      analyzer.PointsTableParameter.ActualName = "Hyperparameter table";
    147156      analyzer.QualityGradientsTableParameter.ActualName = "Gradients table";
    … …  
    152161      finalModelCreator.MeanFunctionParameter.ActualName = MeanFunctionParameterName;
    153162      finalModelCreator.CovarianceFunctionParameter.ActualName = CovarianceFunctionParameterName;
    154       finalModelCreator.HyperparameterParameter.ActualName = initializer.PointParameter.ActualName;
     163      finalModelCreator.HyperparameterParameter.ActualName = bfgsInitializer.PointParameter.ActualName;
    155164      finalModelCreator.Successor = finalAnalyzer;
    156165

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessRegressionModelCreator.cs

    r8375 r8396  
    2525using HeuristicLab.Core;
    2626using HeuristicLab.Data;
     27using HeuristicLab.Encodings.RealVectorEncoding;
    2728using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2829using HeuristicLab.Operators;
    … …  
    6566      ModelParameter.ActualValue = model;
    6667      NegativeLogLikelihoodParameter.ActualValue = new DoubleValue(model.NegativeLogLikelihood);
    67       HyperparameterGradientsParameter.ActualValue = new DoubleArray(model.GetHyperparameterGradients());
     68      HyperparameterGradientsParameter.ActualValue = new RealVector(model.GetHyperparameterGradients());
    6869      return base.Apply();
    6970    }

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/LbfgsAnalyzer.cs

    r8375 r8396  
    2525using HeuristicLab.Core;
    2626using HeuristicLab.Data;
     27using HeuristicLab.Encodings.RealVectorEncoding;
    2728using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2829using HeuristicLab.Operators;
    … …  
    3435namespace HeuristicLab.Algorithms.DataAnalysis {
    3536  [StorableClass]
    36   [Item(Name = "BFGSAnalyzer", Description = "Analyzer to collect results for the BFGS algorithm.")]
    37   public sealed class BFGSAnalyzer : SingleSuccessorOperator, IAnalyzer {
     37  [Item(Name = "LBFGS Analyzer", Description = "Analyzer to collect results for the LM-BFGS algorithm.")]
     38  public sealed class LbfgsAnalyzer : SingleSuccessorOperator, IAnalyzer {
    3839    private const string PointParameterName = "Point";
    3940    private const string QualityGradientsParameterName = "QualityGradients";
    … …  
    4344    private const string PointsTableParameterName = "PointTable";
    4445    private const string QualityGradientsTableParameterName = "QualityGradientsTable";
    45     private const string BFGSStateParameterName = "BFGSState";
     46    private const string StateParameterName = "State";
     47    private const string ApproximateGradientsParameterName = "ApproximateGradients";
    4648
    4749    #region Parameter Properties
    48     public ILookupParameter<DoubleArray> QualityGradientsParameter {
    49       get { return (ILookupParameter<DoubleArray>)Parameters[QualityGradientsParameterName]; }
     50    public ILookupParameter<RealVector> QualityGradientsParameter {
     51      get { return (ILookupParameter<RealVector>)Parameters[QualityGradientsParameterName]; }
    5052    }
    51     public ILookupParameter<DoubleArray> PointParameter {
    52       get { return (ILookupParameter<DoubleArray>)Parameters[PointParameterName]; }
     53    public ILookupParameter<RealVector> PointParameter {
     54      get { return (ILookupParameter<RealVector>)Parameters[PointParameterName]; }
    5355    }
    5456    public ILookupParameter<DoubleValue> QualityParameter {
    … …  
    6769      get { return (ILookupParameter<DataTable>)Parameters[QualityGradientsTableParameterName]; }
    6870    }
    69     public ILookupParameter<BFGSState> BFGSStateParameter {
    70       get { return (ILookupParameter<BFGSState>)Parameters[BFGSStateParameterName]; }
     71    public ILookupParameter<LbfgsState> StateParameter {
     72      get { return (ILookupParameter<LbfgsState>)Parameters[StateParameterName]; }
     73    }
     74    public ILookupParameter<BoolValue> ApproximateGradientsParameter {
     75      get { return (ILookupParameter<BoolValue>)Parameters[ApproximateGradientsParameterName]; }
    7176    }
    7277    #endregion
    7378
    7479    #region Properties
    75     private DoubleArray QualityGradients { get { return QualityGradientsParameter.ActualValue; } }
    76     private DoubleArray Point { get { return PointParameter.ActualValue; } }
     80    private RealVector QualityGradients { get { return QualityGradientsParameter.ActualValue; } }
     81    private RealVector Point { get { return PointParameter.ActualValue; } }
    7782    private DoubleValue Quality { get { return QualityParameter.ActualValue; } }
    7883    private ResultCollection ResultCollection { get { return ResultCollectionParameter.ActualValue; } }
     84    private BoolValue ApproximateGradients { get { return ApproximateGradientsParameter.ActualValue; } }
    7985
    8086    public bool EnabledByDefault {
    … …  
    8591
    8692    [StorableConstructor]
    87     private BFGSAnalyzer(bool deserializing) : base(deserializing) { }
    88     private BFGSAnalyzer(BFGSAnalyzer original, Cloner cloner) : base(original, cloner) { }
    89     public BFGSAnalyzer()
     93    private LbfgsAnalyzer(bool deserializing) : base(deserializing) { }
     94    private LbfgsAnalyzer(LbfgsAnalyzer original, Cloner cloner) : base(original, cloner) { }
     95    public LbfgsAnalyzer()
    9096      : base() {
    9197      // in
    92       Parameters.Add(new LookupParameter<DoubleArray>(PointParameterName, "The current point of the function to optimize."));
    93       Parameters.Add(new LookupParameter<DoubleArray>(QualityGradientsParameterName, "The current gradients of the function to optimize."));
     98      Parameters.Add(new LookupParameter<RealVector>(PointParameterName, "The current point of the function to optimize."));
     99      Parameters.Add(new LookupParameter<RealVector>(QualityGradientsParameterName, "The current gradients of the function to optimize."));
    94100      Parameters.Add(new LookupParameter<DoubleValue>(QualityParameterName, "The current value of the function to optimize."));
    95101      Parameters.Add(new LookupParameter<DataTable>(QualitiesTableParameterName, "The table of all visited quality values."));
    96102      Parameters.Add(new LookupParameter<DataTable>(PointsTableParameterName, "The table of all visited points."));
    97103      Parameters.Add(new LookupParameter<DataTable>(QualityGradientsTableParameterName, "The table of all visited gradient values."));
    98       Parameters.Add(new LookupParameter<BFGSState>(BFGSStateParameterName, "The state of the BFGS optimization algorithm."));
     104      Parameters.Add(new LookupParameter<LbfgsState>(StateParameterName, "The state of the LM-BFGS optimization algorithm."));
     105      Parameters.Add(new LookupParameter<BoolValue>(ApproximateGradientsParameterName,
     106                                              "Flag that indicates if gradients should be approximated."));
     107
    99108      // in & out
    100109      Parameters.Add(new LookupParameter<ResultCollection>(ResultCollectionParameterName, "The result collection of the algorithm."));
    … …  
    102111
    103112    public override IDeepCloneable Clone(Cloner cloner) {
    104       return new BFGSAnalyzer(this, cloner);
     113      return new LbfgsAnalyzer(this, cloner);
    105114    }
    106115
    107116    public override IOperation Apply() {
    108       if (BFGSStateParameter.ActualValue.State.xupdated) {
     117      if (StateParameter.ActualValue.State.xupdated) {
    109118        var f = Quality.Value;
    110         var g = QualityGradients.ToArray();
     119        double[] g;
     120        if (ApproximateGradients.Value) {
     121          g = StateParameter.ActualValue.State.g;
     122        } else {
     123          g = QualityGradients.ToArray();
     124        }
    111125        var x = Point.ToArray();
    112126        var resultCollection = ResultCollection;

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/LbfgsInitializer.cs

    r8375 r8396  
    2525using HeuristicLab.Core;
    2626using HeuristicLab.Data;
     27using HeuristicLab.Encodings.RealVectorEncoding;
    2728using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2829using HeuristicLab.Operators;
    … …  
    3334namespace HeuristicLab.Algorithms.DataAnalysis {
    3435  [StorableClass]
    35   [Item(Name = "BFGSInitializer", Description = "Initializes the necessary data structures for the BFGS algorithm.")]
    36   public sealed class BFGSInitializer : SingleSuccessorOperator {
    37     private const string DimensionParameterName = "Dimension";
     36  [Item(Name = "LBFGS Initializer", Description = "Initializes the necessary data structures for the LM-BFGS algorithm.")]
     37  public sealed class LbfgsInitializer : SingleSuccessorOperator {
    3838    private const string PointParameterName = "Point";
    39     private const string BFGSStateParameterName = "BFGSState";
     39    private const string StateParameterName = "State";
    4040    private const string IterationsParameterName = "Iterations";
     41    private const string ApproximateGradientsParameterName = "ApproximateGradients";
    4142
    4243    #region Parameter Properties
    4344    // in
    44     public ILookupParameter<IntValue> DimensionParameter {
    45       get { return (ILookupParameter<IntValue>)Parameters[DimensionParameterName]; }
    46     }
    4745    public ILookupParameter<IntValue> IterationsParameter {
    4846      get { return (ILookupParameter<IntValue>)Parameters[IterationsParameterName]; }
    4947    }
     48    public ILookupParameter<RealVector> PointParameter {
     49      get { return (ILookupParameter<RealVector>)Parameters[PointParameterName]; }
     50    }
    5051    // out
    51     public ILookupParameter<DoubleArray> PointParameter {
    52       get { return (ILookupParameter<DoubleArray>)Parameters[PointParameterName]; }
     52    public ILookupParameter<LbfgsState> StateParameter {
     53      get { return (ILookupParameter<LbfgsState>)Parameters[StateParameterName]; }
    5354    }
    54     public ILookupParameter<BFGSState> BFGSStateParameter {
    55       get { return (ILookupParameter<BFGSState>)Parameters[BFGSStateParameterName]; }
     55    public ILookupParameter<BoolValue> ApproximateGradientsParameter {
     56      get { return (ILookupParameter<BoolValue>)Parameters[ApproximateGradientsParameterName]; }
    5657    }
    5758
    … …  
    6061
    6162    #region Properties
    62     private IntValue Dimension { get { return DimensionParameter.ActualValue; } }
     63    private RealVector Point { get { return PointParameter.ActualValue; } }
    6364    private IntValue Iterations { get { return IterationsParameter.ActualValue; } }
     65    private BoolValue ApproximateGradients { get { return ApproximateGradientsParameter.ActualValue; } }
    6466    #endregion
    6567
    6668    [StorableConstructor]
    67     private BFGSInitializer(bool deserializing) : base(deserializing) { }
    68     private BFGSInitializer(BFGSInitializer original, Cloner cloner) : base(original, cloner) { }
    69     public BFGSInitializer()
     69    private LbfgsInitializer(bool deserializing) : base(deserializing) { }
     70    private LbfgsInitializer(LbfgsInitializer original, Cloner cloner) : base(original, cloner) { }
     71    public LbfgsInitializer()
    7072      : base() {
    7173      // in
    72       Parameters.Add(new LookupParameter<IntValue>(DimensionParameterName, "The length of the vector to optimize."));
    73       Parameters.Add(new LookupParameter<IntValue>(IterationsParameterName, "The maximal number of iterations for the BFGS algorithm."));
     74      Parameters.Add(new LookupParameter<RealVector>(PointParameterName, "The initial point for the LM-BFGS algorithm."));
     75      Parameters.Add(new LookupParameter<IntValue>(IterationsParameterName, "The maximal number of iterations for the LM-BFGS algorithm."));
     76      Parameters.Add(new LookupParameter<BoolValue>(ApproximateGradientsParameterName,
     77                                                    "Flag that indicates if gradients should be approximated."));
    7478      // out
    75       Parameters.Add(new LookupParameter<DoubleArray>(PointParameterName, "The initial point for the BFGS algorithm."));
    76       Parameters.Add(new LookupParameter<BFGSState>(BFGSStateParameterName, "The state of the BFGS algorithm."));
     79      Parameters.Add(new LookupParameter<LbfgsState>(StateParameterName, "The state of the LM-BFGS algorithm."));
    7780    }
    7881
    7982    public override IDeepCloneable Clone(Cloner cloner) {
    80       return new BFGSInitializer(this, cloner);
     83      return new LbfgsInitializer(this, cloner);
    8184    }
    8285
    8386    public override IOperation Apply() {
    84       int n = Dimension.Value;
    85       double[] initialPoint = Enumerable.Repeat(0.0, n).ToArray();
     87      double[] initialPoint = Point.ToArray();
     88      int n = initialPoint.Length;
    8689      alglib.minlbfgs.minlbfgsstate state = new alglib.minlbfgs.minlbfgsstate();
    87       alglib.minlbfgs.minlbfgscreate(n, Math.Min(n, 7), initialPoint, state);
     90      if (ApproximateGradients.Value) {
     91        alglib.minlbfgs.minlbfgscreatef(n, Math.Min(n, 7), initialPoint, 1E-5, state);
     92      } else {
     93        alglib.minlbfgs.minlbfgscreate(n, Math.Min(n, 7), initialPoint, state);
     94      }
    8895      alglib.minlbfgs.minlbfgssetcond(state, 0, 0, 0, Iterations.Value);
    8996      alglib.minlbfgs.minlbfgssetxrep(state, true);
    9097
    91       PointParameter.ActualValue = new DoubleArray(initialPoint);
    92       BFGSStateParameter.ActualValue = new BFGSState(state);
     98      PointParameter.ActualValue = new RealVector(initialPoint);
     99      StateParameter.ActualValue = new LbfgsState(state);
    93100      return base.Apply();
    94101    }

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/LbfgsMakeStep.cs

    r8375 r8396  
    2424using HeuristicLab.Core;
    2525using HeuristicLab.Data;
     26using HeuristicLab.Encodings.RealVectorEncoding;
    2627using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2728using HeuristicLab.Operators;
    … …  
    3233namespace HeuristicLab.Algorithms.DataAnalysis {
    3334  [StorableClass]
    34   [Item(Name = "BFGSMakeStep", Description = "Makes a step in the BFGS optimization algorithm.")]
    35   public sealed class BFGSMakeStep : SingleSuccessorOperator {
     35  [Item(Name = "LBFGS MakeStep", Description = "Makes a step in the LM-BFGS optimization algorithm.")]
     36  public sealed class LbfgsMakeStep : SingleSuccessorOperator {
    3637    private const string TerminationCriterionParameterName = "TerminationCriterion";
    3738    private const string PointParameterName = "Point";
    38     private const string BFGSStateParameterName = "BFGSState";
     39    private const string StateParameterName = "State";
    3940
    4041    #region Parameter Properties
    41     public ILookupParameter<BFGSState> BFGSStateParameter {
    42       get { return (ILookupParameter<BFGSState>)Parameters[BFGSStateParameterName]; }
     42    public ILookupParameter<LbfgsState> StateParameter {
     43      get { return (ILookupParameter<LbfgsState>)Parameters[StateParameterName]; }
    4344    }
    4445    public ILookupParameter<BoolValue> TerminationCriterionParameter {
    4546      get { return (ILookupParameter<BoolValue>)Parameters[TerminationCriterionParameterName]; }
    4647    }
    47     public ILookupParameter<DoubleArray> PointParameter {
    48       get { return (ILookupParameter<DoubleArray>)Parameters[PointParameterName]; }
     48    public ILookupParameter<RealVector> PointParameter {
     49      get { return (ILookupParameter<RealVector>)Parameters[PointParameterName]; }
    4950    }
    5051    #endregion
    … …  
    5253
    5354    #region Properties
    54     private BFGSState BFGSState { get { return BFGSStateParameter.ActualValue; } }
     55    private LbfgsState State { get { return StateParameter.ActualValue; } }
    5556    #endregion
    5657
    5758    [StorableConstructor]
    58     private BFGSMakeStep(bool deserializing) : base(deserializing) { }
    59     private BFGSMakeStep(BFGSMakeStep original, Cloner cloner) : base(original, cloner) { }
    60     public BFGSMakeStep()
     59    private LbfgsMakeStep(bool deserializing) : base(deserializing) { }
     60    private LbfgsMakeStep(LbfgsMakeStep original, Cloner cloner) : base(original, cloner) { }
     61    public LbfgsMakeStep()
    6162      : base() {
    6263      // in & out
    63       Parameters.Add(new LookupParameter<BFGSState>(BFGSStateParameterName, "The state of the BFGS algorithm."));
     64      Parameters.Add(new LookupParameter<LbfgsState>(StateParameterName, "The state of the LM-BFGS algorithm."));
    6465      // out
    65       Parameters.Add(new LookupParameter<BoolValue>(TerminationCriterionParameterName, "The termination criterion indicating that the BFGS optimization algorithm should stop."));
    66       Parameters.Add(new LookupParameter<DoubleArray>(PointParameterName, "The next point that should be evaluated in the BFGS algorithm."));
     66      Parameters.Add(new LookupParameter<BoolValue>(TerminationCriterionParameterName, "The termination criterion indicating that the LM-BFGS optimization algorithm should stop."));
     67      Parameters.Add(new LookupParameter<RealVector>(PointParameterName, "The next point that should be evaluated in the LM-BFGS algorithm."));
    6768    }
    6869
    6970    public override IDeepCloneable Clone(Cloner cloner) {
    70       return new BFGSMakeStep(this, cloner);
     71      return new LbfgsMakeStep(this, cloner);
    7172    }
    7273
    7374    public override IOperation Apply() {
    74       var state = BFGSState;
     75      var state = State;
    7576      bool @continue = alglib.minlbfgs.minlbfgsiteration(state.State);
    7677      TerminationCriterionParameter.ActualValue = new BoolValue(!@continue);
    7778      if (@continue) {
    78         PointParameter.ActualValue = new DoubleArray(state.State.x);
     79        PointParameter.ActualValue = new RealVector(state.State.x);
    7980      } else {
    8081        double[] x = new double[state.State.x.Length];
    8182        alglib.minlbfgs.minlbfgsreport rep = new alglib.minlbfgs.minlbfgsreport();
    8283        alglib.minlbfgs.minlbfgsresults(state.State, ref x, rep);
    83         PointParameter.ActualValue = new DoubleArray(x);
     84        PointParameter.ActualValue = new RealVector(x);
    8485      }
    8586      return base.Apply();

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/LbfgsState.cs

    r8372 r8396  
    3434namespace HeuristicLab.Algorithms.DataAnalysis {
    3535  [StorableClass]
    36   [Item("BFGSState", "Internal state for the BFGS optimization algorithm.")]
    37   public sealed class BFGSState : Item {
     36  [Item("LbfgsState", "Internal state for the limited-memory BFGS optimization algorithm.")]
     37  public sealed class LbfgsState : Item {
    3838    private alglib.minlbfgs.minlbfgsstate state;
    3939    public alglib.minlbfgs.minlbfgsstate State { get { return state; } }
    4040
    4141    [StorableConstructor]
    42     private BFGSState(bool deserializing)
     42    private LbfgsState(bool deserializing)
    4343      : base(deserializing) {
    4444      state = new alglib.minlbfgs.minlbfgsstate();
    4545    }
    46     private BFGSState(BFGSState original, Cloner cloner)
     46    private LbfgsState(LbfgsState original, Cloner cloner)
    4747      : base(original, cloner) {
    4848      this.state = new alglib.minlbfgs.minlbfgsstate();
    … …  
    125125    }
    126126
    127     public BFGSState(alglib.minlbfgs.minlbfgsstate state)
     127    public LbfgsState(alglib.minlbfgs.minlbfgsstate state)
    128128      : base() {
    129129      this.state = state;
    … …  
    131131
    132132    public override IDeepCloneable Clone(Cloner cloner) {
    133       return new BFGSState(this, cloner);
     133      return new LbfgsState(this, cloner);
    134134    }
    135135

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/LbfgsUpdateResults.cs

    r8375 r8396  
    2424using HeuristicLab.Core;
    2525using HeuristicLab.Data;
     26using HeuristicLab.Encodings.RealVectorEncoding;
    2627using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2728using HeuristicLab.Operators;
    … …  
    3233namespace HeuristicLab.Algorithms.DataAnalysis {
    3334  [StorableClass]
    34   [Item(Name = "BFGSUpdateResults", Description = "Sets the results (function value and gradients) for the next optimization step in the BFGS algorithm.")]
    35   public sealed class BFGSUpdateResults : SingleSuccessorOperator {
     35  [Item(Name = "LBFGS UpdateResults", Description = "Sets the results (function value and gradients) for the next optimization step in the LM-BFGS algorithm.")]
     36  public sealed class LbfgsUpdateResults : SingleSuccessorOperator {
    3637    private const string QualityGradientsParameterName = "QualityGradients";
    3738    private const string QualityParameterName = "Quality";
    38     private const string BFGSStateParameterName = "BFGSState";
     39    private const string StateParameterName = "State";
     40    private const string ApproximateGradientsParameterName = "ApproximateGradients";
    3941
    4042    #region Parameter Properties
    41     public ILookupParameter<DoubleArray> QualityGradientsParameter {
    42       get { return (ILookupParameter<DoubleArray>)Parameters[QualityGradientsParameterName]; }
     43    public ILookupParameter<BoolValue> ApproximateGradientsParameter {
     44      get { return (ILookupParameter<BoolValue>)Parameters[ApproximateGradientsParameterName]; }
     45    }
     46    public ILookupParameter<RealVector> QualityGradientsParameter {
     47      get { return (ILookupParameter<RealVector>)Parameters[QualityGradientsParameterName]; }
    4348    }
    4449    public ILookupParameter<DoubleValue> QualityParameter {
    4550      get { return (ILookupParameter<DoubleValue>)Parameters[QualityParameterName]; }
    4651    }
    47     public ILookupParameter<BFGSState> BFGSStateParameter {
    48       get { return (ILookupParameter<BFGSState>)Parameters[BFGSStateParameterName]; }
     52    public ILookupParameter<LbfgsState> StateParameter {
     53      get { return (ILookupParameter<LbfgsState>)Parameters[StateParameterName]; }
    4954    }
    5055    #endregion
    5156
    5257    #region Properties
    53     private DoubleArray QualityGradients { get { return QualityGradientsParameter.ActualValue; } }
     58    private BoolValue ApproximateGradients { get { return ApproximateGradientsParameter.ActualValue; } }
     59    private RealVector QualityGradients { get { return QualityGradientsParameter.ActualValue; } }
    5460    private DoubleValue Quality { get { return QualityParameter.ActualValue; } }
    55     private BFGSState BFGSState { get { return BFGSStateParameter.ActualValue; } }
     61    private LbfgsState State { get { return StateParameter.ActualValue; } }
    5662    #endregion
    5763
    5864    [StorableConstructor]
    59     private BFGSUpdateResults(bool deserializing) : base(deserializing) { }
    60     private BFGSUpdateResults(BFGSUpdateResults original, Cloner cloner) : base(original, cloner) { }
    61     public BFGSUpdateResults()
     65    private LbfgsUpdateResults(bool deserializing) : base(deserializing) { }
     66    private LbfgsUpdateResults(LbfgsUpdateResults original, Cloner cloner) : base(original, cloner) { }
     67    public LbfgsUpdateResults()
    6268      : base() {
    6369      // in
    64       Parameters.Add(new LookupParameter<DoubleArray>(QualityGradientsParameterName, "The gradients at the evaluated point of the function to optimize."));
     70      Parameters.Add(new LookupParameter<RealVector>(QualityGradientsParameterName, "The gradients at the evaluated point of the function to optimize."));
    6571      Parameters.Add(new LookupParameter<DoubleValue>(QualityParameterName, "The value at the evaluated point of the function to optimize."));
     72      Parameters.Add(new LookupParameter<BoolValue>(ApproximateGradientsParameterName,
     73                                                    "Flag that indicates if gradients should be approximated."));
    6674      // in & out
    67       Parameters.Add(new LookupParameter<BFGSState>(BFGSStateParameterName, "The state of the BFGS algorithm."));
     75      Parameters.Add(new LookupParameter<LbfgsState>(StateParameterName, "The state of the LM-BFGS algorithm."));
    6876    }
    6977
    7078    public override IDeepCloneable Clone(Cloner cloner) {
    71       return new BFGSUpdateResults(this, cloner);
     79      return new LbfgsUpdateResults(this, cloner);
    7280    }
    7381
    7482    public override IOperation Apply() {
    75       var state = BFGSState;
     83      var state = State;
    7684      var f = Quality.Value;
    77       var g = QualityGradients.ToArray();
    7885      state.State.f = f;
    79       state.State.g = g;
     86      if (!ApproximateGradients.Value) {
     87        var g = QualityGradients.ToArray();
     88        state.State.g = g;
     89      }
    8090      return base.Apply();
    8191    }

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

    r8375 r8396  
    122122    </Compile>
    123123    <Compile Include="FixedDataAnalysisAlgorithm.cs" />
    124     <Compile Include="GaussianProcess\BFGSInitializer.cs" />
    125     <Compile Include="GaussianProcess\BFGSState.cs" />
    126     <Compile Include="GaussianProcess\BFGSUpdateResults.cs" />
    127     <Compile Include="GaussianProcess\BFGSMakeStep.cs" />
    128     <Compile Include="GaussianProcess\BFGSAnalyzer.cs" />
     124    <Compile Include="GaussianProcess\Lbfgs.cs" />
     125    <Compile Include="GaussianProcess\LbfgsInitializer.cs" />
     126    <Compile Include="GaussianProcess\LbfgsState.cs" />
     127    <Compile Include="GaussianProcess\LbfgsUpdateResults.cs" />
     128    <Compile Include="GaussianProcess\LbfgsMakeStep.cs" />
     129    <Compile Include="GaussianProcess\LbfgsAnalyzer.cs" />
     130    <Compile Include="GaussianProcess\GaussianProcessHyperparameterInitializer.cs" />
    129131    <Compile Include="GaussianProcess\GaussianProcessRegressionSolutionCreator.cs" />
    130     <Compile Include="GaussianProcess\GaussianProcessSetHyperparameterLength.cs" />
    131132    <Compile Include="GaussianProcess\GaussianProcessRegressionModelCreator.cs" />
    132133    <Compile Include="GaussianProcess\CovarianceLinear.cs" />
    … …  
    243244      <Private>False</Private>
    244245    </ProjectReference>
     246    <ProjectReference Include="..\..\HeuristicLab.Encodings.RealVectorEncoding\3.3\HeuristicLab.Encodings.RealVectorEncoding-3.3.csproj">
     247      <Project>{BB6D334A-4BB6-4674-9883-31A6EBB32CAB}</Project>
     248      <Name>HeuristicLab.Encodings.RealVectorEncoding-3.3</Name>
     249    </ProjectReference>
    245250    <ProjectReference Include="..\..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
    246251      <Project>{06D4A186-9319-48A0-BADE-A2058D462EEA}</Project>
    … …  
    300305      <Name>HeuristicLab.Problems.Instances-3.3</Name>
    301306      <Private>False</Private>
     307    </ProjectReference>
     308    <ProjectReference Include="..\..\HeuristicLab.Problems.TestFunctions\3.3\HeuristicLab.Problems.TestFunctions-3.3.csproj">
     309      <Project>{88B9B0E3-344E-4196-82A3-0F9732506FE8}</Project>
     310      <Name>HeuristicLab.Problems.TestFunctions-3.3</Name>
    302311    </ProjectReference>
    303312    <ProjectReference Include="..\..\HeuristicLab.Random\3.3\HeuristicLab.Random-3.3.csproj">

  • trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Plugin.cs.frame

    r8375 r8396  
    3737  [PluginDependency("HeuristicLab.Data", "3.3")]
    3838  [PluginDependency("HeuristicLab.Encodings.SymbolicExpressionTreeEncoding", "3.4")]
     39  [PluginDependency("HeuristicLab.Encodings.RealVectorEncoding", "3.3")]
    3940  [PluginDependency("HeuristicLab.Operators", "3.3")]
    4041  [PluginDependency("HeuristicLab.Optimization", "3.3")]

  • trunk/sources/HeuristicLab.Algorithms.GeneticAlgorithm/3.3/Plugin.cs.frame

    r8246 r8396  
    4040  [PluginDependency("HeuristicLab.Random", "3.3")]
    4141  [PluginDependency("HeuristicLab.Selection", "3.3")]
     42  [PluginDependency("HeuristicLab.Problems.TestFunctions", "3.3")]
    4243  public class HeuristicLabAlgorithmsGeneticAlgorithmPlugin : PluginBase {
    4344  }
Note: See TracChangeset for help on using the changeset viewer.