Index: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/Friedman/FriedmanRandomFunction.cs
===================================================================
--- trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/Friedman/FriedmanRandomFunction.cs	(revision 14623)
+++ trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/Friedman/FriedmanRandomFunction.cs	(revision 14630)
@@ -94,5 +94,5 @@
     }
 
-    // as described in Greedy Function Approxination paper
+    // as described in Greedy Function Approximation paper
     private IEnumerable<double> GenerateRandomFunction(IRandom rand, List<List<double>> xs, int nTerms = 20) {
       int nRows = xs.First().Count;
Index: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/GaussianProcessVariableNetwork.cs
===================================================================
--- trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/GaussianProcessVariableNetwork.cs	(revision 14630)
+++ trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/GaussianProcessVariableNetwork.cs	(revision 14630)
@@ -0,0 +1,133 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Problems.Instances.DataAnalysis {
+  public sealed class GaussianProcessVariableNetwork : VariableNetwork {
+    private int numberOfFeatures;
+    private double noiseRatio;
+
+    public override string Name { get { return string.Format("GaussianProcessVariableNetwork-{0:0%} ({1} dim)", noiseRatio, numberOfFeatures); } }
+
+    public GaussianProcessVariableNetwork(int numberOfFeatures, double noiseRatio,
+      IRandom rand)
+      : base(250, 250, numberOfFeatures, noiseRatio, rand) {
+      this.noiseRatio = noiseRatio;
+      this.numberOfFeatures = numberOfFeatures;
+    }
+
+    // sample the input variables that are actually used and sample from a Gaussian process
+    protected override IEnumerable<double> GenerateRandomFunction(IRandom rand, List<List<double>> xs, out string[] selectedVarNames, out double[] relevance) {
+      int nl = SampleNumberOfVariables(rand, xs.Count);
+
+      var selectedIdx = Enumerable.Range(0, xs.Count).Shuffle(rand)
+        .Take(nl).ToArray();
+
+      var selectedVars = selectedIdx.Select(i => xs[i]).ToArray();
+      selectedVarNames = selectedIdx.Select(i => VariableNames[i]).ToArray();
+      return SampleGaussianProcess(rand, selectedVars, out relevance);
+    }
+
+    private IEnumerable<double> SampleGaussianProcess(IRandom rand, List<double>[] xs, out double[] relevance) {
+      int nl = xs.Length;
+      int nRows = xs.First().Count;
+
+      // sample u iid ~ N(0, 1)
+      var u = Enumerable.Range(0, nRows).Select(_ => NormalDistributedRandom.NextDouble(rand, 0, 1)).ToArray();
+
+      // sample actual length-scales
+      var l = Enumerable.Range(0, nl)
+        .Select(_ => rand.NextDouble() * 2 + 0.5)
+        .ToArray();
+
+      double[,] K = CalculateCovariance(xs, l);
+
+      // decompose
+      alglib.trfac.spdmatrixcholesky(ref K, nRows, false);
+
+
+      // calc y = Lu
+      var y = new double[u.Length];
+      alglib.ablas.rmatrixmv(nRows, nRows, K, 0, 0, 0, u, 0, ref y, 0);
+
+      // calculate relevance by removing dimensions
+      relevance = CalculateRelevance(y, u, xs, l);
+
+      return y;
+    }
+
+    // calculate variable relevance based on removal of variables
+    //  1) to remove a variable we set it's length scale to infinity (no relation of the variable value to the target)
+    //  2) calculate MSE of the original target values (y) to the updated targes y' (after variable removal)
+    //  3) relevance is larger if MSE(y,y') is large
+    //  4) scale impacts so that the most important variable has impact = 1
+    private double[] CalculateRelevance(double[] y, double[] u, List<double>[] xs, double[] l) {
+      int nRows = xs.First().Count;
+      var changedL = new double[l.Length];
+      var relevance = new double[l.Length];
+      for(int i = 0; i < l.Length; i++) {
+        Array.Copy(l, changedL, changedL.Length);
+        changedL[i] = double.MaxValue;
+        var changedK = CalculateCovariance(xs, changedL);
+
+        var yChanged = new double[u.Length];
+        alglib.ablas.rmatrixmv(nRows, nRows, changedK, 0, 0, 0, u, 0, ref yChanged, 0);
+
+        OnlineCalculatorError error;
+        var mse = OnlineMeanSquaredErrorCalculator.Calculate(y, yChanged, out error);
+        if(error != OnlineCalculatorError.None) mse = double.MaxValue;
+        relevance[i] = mse;
+      }
+      // scale so that max relevance is 1.0
+      var maxRel = relevance.Max();
+      for(int i = 0; i < relevance.Length; i++) relevance[i] /= maxRel;
+      return relevance;
+    }
+
+    private double[,] CalculateCovariance(List<double>[] xs, double[] l) {
+      int nRows = xs.First().Count;
+      double[,] K = new double[nRows, nRows];
+      for(int r = 0; r < nRows; r++) {
+        double[] xi = xs.Select(x => x[r]).ToArray();
+        for(int c = 0; c <= r; c++) {
+          double[] xj = xs.Select(x => x[c]).ToArray();
+          double dSqr = xi.Zip(xj, (xik, xjk) => (xik - xjk))
+            .Select(dk => dk * dk)
+            .Zip(l, (dk, lk) => dk / lk)
+            .Sum();
+          K[r, c] = Math.Exp(-dSqr);
+        }
+      }
+      // add a small diagonal matrix for numeric stability
+      for(int i = 0; i < nRows; i++) {
+        K[i, i] += 1.0E-7;
+      }
+
+      return K;
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/LinearVariableNetwork.cs
===================================================================
--- trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/LinearVariableNetwork.cs	(revision 14630)
+++ trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/LinearVariableNetwork.cs	(revision 14630)
@@ -0,0 +1,109 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Problems.Instances.DataAnalysis {
+  public sealed class LinearVariableNetwork : VariableNetwork {
+    private int numberOfFeatures;
+    private double noiseRatio;
+
+    public override string Name { get { return string.Format("LinearVariableNetwork-{0:0%} ({1} dim)", noiseRatio, numberOfFeatures); } }
+
+    public LinearVariableNetwork(int numberOfFeatures, double noiseRatio,
+      IRandom rand)
+      : base(250, 250, numberOfFeatures, noiseRatio, rand) {
+      this.noiseRatio = noiseRatio;
+      this.numberOfFeatures = numberOfFeatures;
+    }
+
+    protected override IEnumerable<double> GenerateRandomFunction(IRandom rand, List<List<double>> xs, out string[] selectedVarNames, out double[] relevance) {
+      int nl = SampleNumberOfVariables(rand, numberOfFeatures);
+
+      var selectedIdx = Enumerable.Range(0, xs.Count).Shuffle(rand)
+        .Take(nl).ToArray();
+
+      var selectedVars = selectedIdx.Select(i => xs[i]).ToArray();
+      selectedVarNames = selectedIdx.Select(i => VariableNames[i]).ToArray();
+      return SampleLinearFunction(rand, selectedVars, out relevance);
+    }
+
+    private IEnumerable<double> SampleLinearFunction(IRandom rand, List<double>[] xs, out double[] relevance) {
+      int nl = xs.Length;
+      int nRows = xs.First().Count;
+
+      // sample standardized coefficients iid ~ N(0, 1)
+      var c = Enumerable.Range(0, nRows).Select(_ => NormalDistributedRandom.NextDouble(rand, 0, 1)).ToArray();
+
+      // calculate scaled coefficients (variables with large variance should have smaller coefficients)
+      var scaledC = Enumerable.Range(0, nl)
+        .Select(i => c[i] / xs[i].StandardDeviationPop())
+        .ToArray();
+
+      var y = EvaluteLinearModel(xs, scaledC);
+
+      relevance = CalculateRelevance(y, xs, scaledC);
+
+      return y;
+    }
+
+    private double[] EvaluteLinearModel(List<double>[] xs, double[] c) {
+      int nRows = xs.First().Count;
+      var y = new double[nRows];
+      for(int row = 0; row < nRows; row++) {
+        y[row] = xs.Select(xi => xi[row]).Zip(c, (xij, cj) => xij * cj).Sum();
+        y[row] /= c.Length;
+      }
+      return y;
+    }
+
+    // calculate variable relevance based on removal of variables
+    //  1) to remove a variable we set it's coefficient to zero
+    //  2) calculate MSE of the original target values (y) to the updated targes y' (after variable removal)
+    //  3) relevance is larger if MSE(y,y') is large
+    //  4) scale impacts so that the most important variable has impact = 1
+    private double[] CalculateRelevance(double[] y, List<double>[] xs, double[] l) {
+      var changedL = new double[l.Length];
+      var relevance = new double[l.Length];
+      for(int i = 0; i < l.Length; i++) {
+        Array.Copy(l, changedL, changedL.Length);
+        changedL[i] = 0.0;
+
+        var yChanged = EvaluteLinearModel(xs, changedL);
+
+        OnlineCalculatorError error;
+        var mse = OnlineMeanSquaredErrorCalculator.Calculate(y, yChanged, out error);
+        if(error != OnlineCalculatorError.None) mse = double.MaxValue;
+        relevance[i] = mse;
+      }
+      // scale so that max relevance is 1.0
+      var maxRel = relevance.Max();
+      for(int i = 0; i < relevance.Length; i++) relevance[i] /= maxRel;
+      return relevance;
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetwork.cs
===================================================================
--- trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetwork.cs	(revision 14623)
+++ trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetwork.cs	(revision 14630)
@@ -30,5 +30,5 @@
 
 namespace HeuristicLab.Problems.Instances.DataAnalysis {
-  public class VariableNetwork : ArtificialRegressionDataDescriptor {
+  public abstract class VariableNetwork : ArtificialRegressionDataDescriptor {
     private int nTrainingSamples;
     private int nTestSamples;
@@ -38,5 +38,4 @@
     private IRandom random;
 
-    public override string Name { get { return string.Format("VariableNetwork-{0:0%} ({1} dim)", noiseRatio, numberOfFeatures); } }
     private string networkDefinition;
     public string NetworkDefinition { get { return networkDefinition; } }
@@ -47,9 +46,5 @@
     }
 
-    public VariableNetwork(int numberOfFeatures, double noiseRatio,
-      IRandom rand)
-      : this(250, 250, numberOfFeatures, noiseRatio, rand) { }
-
-    public VariableNetwork(int nTrainingSamples, int nTestSamples,
+    protected VariableNetwork(int nTrainingSamples, int nTestSamples,
       int numberOfFeatures, double noiseRatio, IRandom rand) {
       this.nTrainingSamples = nTrainingSamples;
@@ -105,9 +100,15 @@
 
       var nrand = new NormalDistributedRandom(random, 0, 1);
-      for (int c = 0; c < numLvl0; c++) {
+      for(int c = 0; c < numLvl0; c++) {
         inputVarNames.Add(new string[] { });
         relevances.Add(new double[] { });
-        description.Add(" ~ N(0, 1)");
-        lvl0.Add(Enumerable.Range(0, TestPartitionEnd).Select(_ => nrand.NextDouble()).ToList());
+        description.Add(" ~ N(0, 1 + noiseLvl)");
+        // use same generation procedure for all variables
+        var x = Enumerable.Range(0, TestPartitionEnd).Select(_ => nrand.NextDouble()).ToList();
+        var sigma = x.StandardDeviationPop();
+        var mean = x.Average();
+        for(int i = 0; i < x.Count; i++) x[i] = (x[i] - mean) / sigma;
+        var noisePrng = new NormalDistributedRandom(random, 0, Math.Sqrt(noiseRatio / (1.0 - noiseRatio)));
+        lvl0.Add(x.Select(t => t + noisePrng.NextDouble()).ToList());
       }
 
@@ -125,5 +126,5 @@
 
       this.variableRelevances.Clear();
-      for (int i = 0; i < variableNames.Length; i++) {
+      for(int i = 0; i < variableNames.Length; i++) {
         var targetVarName = variableNames[i];
         var targetRelevantInputs =
@@ -136,9 +137,9 @@
       // for graphviz
       networkDefinition += Environment.NewLine + "digraph G {";
-      for (int i = 0; i < variableNames.Length; i++) {
+      for(int i = 0; i < variableNames.Length; i++) {
         var name = variableNames[i];
         var selectedVarNames = inputVarNames[i];
         var selectedRelevances = relevances[i];
-        for (int j = 0; j < selectedVarNames.Length; j++) {
+        for(int j = 0; j < selectedVarNames.Length; j++) {
           var selectedVarName = selectedVarNames[j];
           var selectedRelevance = selectedRelevances[j];
@@ -157,12 +158,16 @@
 
     private List<List<double>> CreateVariables(List<List<double>> allowedInputs, int numVars, List<string[]> inputVarNames, List<string> description, List<double[]> relevances) {
-      var res = new List<List<double>>();
-      for (int c = 0; c < numVars; c++) {
+      var newVariables = new List<List<double>>();
+      for(int c = 0; c < numVars; c++) {
         string[] selectedVarNames;
         double[] relevance;
-        var x = GenerateRandomFunction(random, allowedInputs, out selectedVarNames, out relevance);
+        var x = GenerateRandomFunction(random, allowedInputs, out selectedVarNames, out relevance).ToArray();
+        // standardize x
         var sigma = x.StandardDeviation();
-        var noisePrng = new NormalDistributedRandom(random, 0, sigma * Math.Sqrt(noiseRatio / (1.0 - noiseRatio)));
-        res.Add(x.Select(t => t + noisePrng.NextDouble()).ToList());
+        var mean = x.Average();
+        for(int i = 0; i < x.Length; i++) x[i] = (x[i] - mean) / sigma;
+
+        var noisePrng = new NormalDistributedRandom(random, 0, Math.Sqrt(noiseRatio / (1.0 - noiseRatio)));
+        newVariables.Add(x.Select(t => t + noisePrng.NextDouble()).ToList());
         Array.Sort(selectedVarNames, relevance);
         inputVarNames.Add(selectedVarNames);
@@ -176,98 +181,15 @@
         description.Add(string.Format(" ~ N({0}, {1:N3}) [Relevances: {2}]", desc, noisePrng.Sigma, relevanceStr));
       }
-      return res;
+      return newVariables;
     }
 
-    // sample the input variables that are actually used and sample from a Gaussian process
-    private IEnumerable<double> GenerateRandomFunction(IRandom rand, List<List<double>> xs, out string[] selectedVarNames, out double[] relevance) {
+    public int SampleNumberOfVariables(IRandom rand, int maxNumberOfVariables) {
       double r = -Math.Log(1.0 - rand.NextDouble()) * 2.0; // r is exponentially distributed with lambda = 2
       int nl = (int)Math.Floor(1.5 + r); // number of selected vars is likely to be between three and four
-      if (nl > xs.Count) nl = xs.Count; // limit max
-
-      var selectedIdx = Enumerable.Range(0, xs.Count).Shuffle(random)
-        .Take(nl).ToArray();
-
-      var selectedVars = selectedIdx.Select(i => xs[i]).ToArray();
-      selectedVarNames = selectedIdx.Select(i => VariableNames[i]).ToArray();
-      return SampleGaussianProcess(random, selectedVars, out relevance);
+      return Math.Min(maxNumberOfVariables, nl);
     }
 
-    private IEnumerable<double> SampleGaussianProcess(IRandom random, List<double>[] xs, out double[] relevance) {
-      int nl = xs.Length;
-      int nRows = xs.First().Count;
-
-      // sample u iid ~ N(0, 1)
-      var u = Enumerable.Range(0, nRows).Select(_ => NormalDistributedRandom.NextDouble(random, 0, 1)).ToArray();
-
-      // sample actual length-scales
-      var l = Enumerable.Range(0, nl)
-        .Select(_ => random.NextDouble() * 2 + 0.5)
-        .ToArray();
-
-      double[,] K = CalculateCovariance(xs, l);
-
-      // decompose
-      alglib.trfac.spdmatrixcholesky(ref K, nRows, false);
-
-
-      // calc y = Lu
-      var y = new double[u.Length];
-      alglib.ablas.rmatrixmv(nRows, nRows, K, 0, 0, 0, u, 0, ref y, 0);
-
-      // calculate relevance by removing dimensions
-      relevance = CalculateRelevance(y, u, xs, l);
-
-      return y;
-    }
-
-    // calculate variable relevance based on removal of variables
-    //  1) to remove a variable we set it's length scale to infinity (no relation of the variable value to the target)
-    //  2) calculate MSE of the original target values (y) to the updated targes y' (after variable removal)
-    //  3) relevance is larger if MSE(y,y') is large
-    //  4) scale impacts so that the most important variable has impact = 1
-    private double[] CalculateRelevance(double[] y, double[] u, List<double>[] xs, double[] l) {
-      int nRows = xs.First().Count;
-      var changedL = new double[l.Length];
-      var relevance = new double[l.Length];
-      for (int i = 0; i < l.Length; i++) {
-        Array.Copy(l, changedL, changedL.Length);
-        changedL[i] = double.MaxValue;
-        var changedK = CalculateCovariance(xs, changedL);
-
-        var yChanged = new double[u.Length];
-        alglib.ablas.rmatrixmv(nRows, nRows, changedK, 0, 0, 0, u, 0, ref yChanged, 0);
-
-        OnlineCalculatorError error;
-        var mse = OnlineMeanSquaredErrorCalculator.Calculate(y, yChanged, out error);
-        if (error != OnlineCalculatorError.None) mse = double.MaxValue;
-        relevance[i] = mse;
-      }
-      // scale so that max relevance is 1.0
-      var maxRel = relevance.Max();
-      for (int i = 0; i < relevance.Length; i++) relevance[i] /= maxRel;
-      return relevance;
-    }
-
-    private double[,] CalculateCovariance(List<double>[] xs, double[] l) {
-      int nRows = xs.First().Count;
-      double[,] K = new double[nRows, nRows];
-      for (int r = 0; r < nRows; r++) {
-        double[] xi = xs.Select(x => x[r]).ToArray();
-        for (int c = 0; c <= r; c++) {
-          double[] xj = xs.Select(x => x[c]).ToArray();
-          double dSqr = xi.Zip(xj, (xik, xjk) => (xik - xjk))
-            .Select(dk => dk * dk)
-            .Zip(l, (dk, lk) => dk / lk)
-            .Sum();
-          K[r, c] = Math.Exp(-dSqr);
-        }
-      }
-      // add a small diagonal matrix for numeric stability
-      for (int i = 0; i < nRows; i++) {
-        K[i, i] += 1.0E-7;
-      }
-
-      return K;
-    }
+    // sample a random function and calculate the variable relevances
+    protected abstract IEnumerable<double> GenerateRandomFunction(IRandom rand, List<List<double>> xs, out string[] selectedVarNames, out double[] relevance);
   }
 }
Index: trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetworkInstanceProvider.cs
===================================================================
--- trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetworkInstanceProvider.cs	(revision 14623)
+++ trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Regression/VariableNetworks/VariableNetworkInstanceProvider.cs	(revision 14630)
@@ -49,11 +49,17 @@
     public override IEnumerable<IDataDescriptor> GetDataDescriptors() {
       var numVariables = new int[] { 10, 20, 50, 100 };
-      var noiseRatios = new double[] { 0, 0.01, 0.05, 0.1 };
+      var noiseRatios = new double[] { 0, 0.01, 0.05, 0.1, 0.2 };
       var rand = new MersenneTwister((uint)Seed); // use fixed seed for deterministic problem generation
-      return (from size in numVariables
-              from noiseRatio in noiseRatios
-              select new VariableNetwork(size, noiseRatio, new MersenneTwister((uint)rand.Next())))
-              .Cast<IDataDescriptor>()
-              .ToList();
+      var lr = (from size in numVariables
+                from noiseRatio in noiseRatios
+                select new LinearVariableNetwork(size, noiseRatio, new MersenneTwister((uint)rand.Next())))
+                .Cast<IDataDescriptor>()
+                .ToList();
+      var gp = (from size in numVariables
+                from noiseRatio in noiseRatios
+                select new GaussianProcessVariableNetwork(size, noiseRatio, new MersenneTwister((uint)rand.Next())))
+                .Cast<IDataDescriptor>()
+                .ToList();
+      return lr.Concat(gp);
     }
 
