Changeset 363

Timestamp:

07/04/08 17:30:24 (16 years ago)

Author:

gkronber

Message:

• implemented operator to store the best of run solution, in regard of a specific fitness variable).
• adapted struct-id infrastructure to allow evaluation of models on validation data.

ticket #194

Location:

trunk/sources

Files:

• HeuristicLab.DataAnalysis/DatasetParser.cs (modified) (2 diffs)
• HeuristicLab.Functions/BakedFunctionTree.cs (modified) (1 diff)
• HeuristicLab.Functions/BakedTreeEvaluator.cs (modified) (2 diffs)
• HeuristicLab.Functions/IFunctionTree.cs (modified) (1 diff)
• HeuristicLab.Logging/BestSolutionStorer.cs (added)
• HeuristicLab.Logging/HeuristicLab.Logging.csproj (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/CoefficientOfDeterminationEvaluator.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/EarlyStoppingMeanSquaredErrorEvaluator.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/GPEvaluatorBase.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/MCCEvaluator.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/MeanSquaredErrorEvaluator.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/Evaluation/SimpleEvaluator.cs (added)
• HeuristicLab.StructureIdentification/Evaluation/VarianceAccountedForEvaluator.cs (modified) (1 diff)
• HeuristicLab.StructureIdentification/HeuristicLab.StructureIdentification.csproj (modified) (1 diff)
• HeuristicLab.StructureIdentification/StructIdProblemInjector.cs (modified) (2 diffs)
• HeuristicLab.StructureIdentification/StructIdProblemInjectorView.Designer.cs (modified) (7 diffs)
• HeuristicLab.StructureIdentification/StructIdProblemInjectorView.cs (modified) (2 diffs)

trunk/sources/HeuristicLab.DataAnalysis/DatasetParser.cs

@@ -35 +35 @@
     private const string TRAININGSAMPLESSTART = "TRAININGSAMPLESSTART";
     private const string TRAININGSAMPLESEND = "TRAININGSAMPLESEND";
+    private const string VALIDATIONSAMPLESSTART = "VALIDATIONSAMPLESSTART";
+    private const string VALIDATIONSAMPLESEND = "VALIDATIONSAMPLESEND";
     private Tokenizer tokenizer;
     private Dictionary<string, List<Token>> metadata;
@@ -121 +123 @@
         if(metadata.ContainsKey(TRAININGSAMPLESEND)) {
           return metadata[TRAININGSAMPLESEND][0].intValue;
+        } else return rows;
+      }
+    }
+    public int ValidationSamplesStart {
+      get {
+        if(metadata.ContainsKey(VALIDATIONSAMPLESSTART)) {
+          return metadata[VALIDATIONSAMPLESSTART][0].intValue;
+        } else return 0;
+      }
+    }
+
+    public int ValidationSamplesEnd {
+      get {
+        if(metadata.ContainsKey(VALIDATIONSAMPLESEND)) {
+          return metadata[VALIDATIONSAMPLESEND][0].intValue;
         } else return rows;
       }

trunk/sources/HeuristicLab.Functions/BakedFunctionTree.cs

@@ -262 +262 @@
     }
 
-    bool evaluatorReset = false;
-    public double Evaluate(Dataset dataset, int sampleIndex) {
+    public void PrepareEvaluation(Dataset dataset) {
       FlattenVariables();
       FlattenTrees();
-      if(!evaluatorReset) {
-        BakedTreeEvaluator.ResetEvaluator(linearRepresentation);
-        evaluatorReset = true;
-      }
-      return BakedTreeEvaluator.Evaluate(dataset, sampleIndex);
+      BakedTreeEvaluator.ResetEvaluator(dataset, linearRepresentation);
+    }
+
+    public double Evaluate(int sampleIndex) {
+      return BakedTreeEvaluator.Evaluate(sampleIndex);
     }
 

trunk/sources/HeuristicLab.Functions/BakedTreeEvaluator.cs

@@ -53 +53 @@
     }
 
-    public static void ResetEvaluator(List<LightWeightFunction> linearRepresentation) {
+    public static void ResetEvaluator(Dataset dataset, List<LightWeightFunction> linearRepresentation) {
       int i = 0;
+      BakedTreeEvaluator.dataset = dataset;
       foreach(LightWeightFunction f in linearRepresentation) {
         TranslateToInstr(f, codeArr[i++]);
@@ -78 +79 @@
     }
 
-    internal static double Evaluate(Dataset dataset, int sampleIndex) {
+    internal static double Evaluate(int sampleIndex) {
       PC = 0;
       BakedTreeEvaluator.sampleIndex = sampleIndex;
-      BakedTreeEvaluator.dataset = dataset;
       return EvaluateBakedCode();
     }

trunk/sources/HeuristicLab.Functions/IFunctionTree.cs

@@ -40 +40 @@
     void RemoveSubTree(int index);
 
-    double Evaluate(Dataset dataset, int sampleIndex);
+    void PrepareEvaluation(Dataset dataset);
+    double Evaluate(int sampleIndex);
   }
 }

trunk/sources/HeuristicLab.Logging/HeuristicLab.Logging.csproj

@@ -46 +46 @@
   <ItemGroup>
     <Compile Include="BestAverageWorstQualityCalculator.cs" />
+    <Compile Include="BestSolutionStorer.cs" />
     <Compile Include="Logger.cs" />
     <Compile Include="Linechart.cs" />

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/CoefficientOfDeterminationEvaluator.cs

@@ -47 +47 @@
       double originalDeviationTotalSumOfSquares = 0.0;
       double targetMean = dataset.GetMean(targetVariable);
+      functionTree.PrepareEvaluation(dataset);
       for(int sample = 0; sample < dataset.Rows; sample++) {
-        double estimated = functionTree.Evaluate(dataset, sample);
+        double estimated = functionTree.Evaluate(sample);
         double original = dataset.GetValue(sample, targetVariable);
         if(!double.IsNaN(original) && !double.IsInfinity(original)) {

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/EarlyStoppingMeanSquaredErrorEvaluator.cs

@@ -60 +60 @@
       double errorsSquaredSum = 0;
       double targetMean = dataset.GetMean(targetVariable);
+      functionTree.PrepareEvaluation(dataset);
       for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = functionTree.Evaluate(dataset, sample);
+        double estimated = functionTree.Evaluate(sample);
         double original = dataset.GetValue(sample, targetVariable);
         if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/GPEvaluatorBase.cs

@@ -57 +57 @@
       this.totalEvaluatedNodes = scope.GetVariableValue<DoubleData>("TotalEvaluatedNodes", true).Data;
       double result = Evaluate(scope, functionTree, targetVariable, dataset);
-      scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("Quality"), new DoubleData(result)));
+
+      DoubleData quality = GetVariableValue<DoubleData>("Quality", scope, false, false);
+      if(quality == null) {
+        scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName("Quality"), new DoubleData(result)));
+      } else {
+        quality.Data = result;
+      }
+
       return null;
     }

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MCCEvaluator.cs

@@ -54 +54 @@
       double negative = 0;
       double targetMean = dataset.GetMean(targetVariable);
+      functionTree.PrepareEvaluation(dataset);
       for(int sample = 0; sample < dataset.Rows; sample++) {
-        double est = functionTree.Evaluate(dataset, sample);
+        double est = functionTree.Evaluate(sample);
         double orig = dataset.GetValue(sample, targetVariable);
         if(double.IsNaN(est) || double.IsInfinity(est)) {

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/MeanSquaredErrorEvaluator.cs

@@ -60 +60 @@
       }
 
+      functionTree.PrepareEvaluation(dataset);
       for(int sample = trainingStart; sample < trainingEnd; sample++) {
-        double estimated = functionTree.Evaluate(dataset, sample);
+        double estimated = functionTree.Evaluate(sample);
         double original = dataset.GetValue(sample, targetVariable);
         if(double.IsNaN(estimated) || double.IsInfinity(estimated)) {

trunk/sources/HeuristicLab.StructureIdentification/Evaluation/VarianceAccountedForEvaluator.cs

@@ -57 +57 @@
       double[] originalTargetVariableValues = new double[dataset.Rows];
       double targetMean = dataset.GetMean(targetVariable);
+      functionTree.PrepareEvaluation(dataset);
       for(int sample = 0; sample < dataset.Rows; sample++) {
-        double estimated = functionTree.Evaluate(dataset, sample);
+        double estimated = functionTree.Evaluate(sample);
         double original = dataset.GetValue(sample, targetVariable);
         if(!double.IsNaN(original) && !double.IsInfinity(original)) {

trunk/sources/HeuristicLab.StructureIdentification/HeuristicLab.StructureIdentification.csproj

@@ -48 +48 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Evaluation\SimpleEvaluator.cs" />
     <Compile Include="ProbabilisticTreeCreator.cs" />
     <Compile Include="Evaluation\CoefficientOfDeterminationEvaluator.cs" />

trunk/sources/HeuristicLab.StructureIdentification/StructIdProblemInjector.cs

@@ -41 +41 @@
       AddVariableInfo(new VariableInfo("TargetVariable", "TargetVariable", typeof(IntData), VariableKind.New));
       AddVariable(new Variable("TargetVariable", new IntData()));
+      AddVariableInfo(new VariableInfo("TrainingSamplesStart", "TrainingSamplesStart", typeof(IntData), VariableKind.New));
+      AddVariable(new Variable("TrainingSamplesStart", new IntData()));
+      AddVariableInfo(new VariableInfo("TrainingSamplesEnd", "TrainingSamplesEnd", typeof(IntData), VariableKind.New));
+      AddVariable(new Variable("TrainingSamplesEnd", new IntData()));
+      AddVariableInfo(new VariableInfo("ValidationSamplesStart", "ValidationSamplesStart", typeof(IntData), VariableKind.New));
+      AddVariable(new Variable("ValidationSamplesStart", new IntData()));
+      AddVariableInfo(new VariableInfo("ValidationSamplesEnd", "ValidationSamplesEnd", typeof(IntData), VariableKind.New));
+      AddVariable(new Variable("ValidationSamplesEnd", new IntData()));
     }
 
@@ -50 +58 @@
       scope.AddVariable(new Variable(scope.TranslateName("Dataset"), (IItem)GetVariable("Dataset").Value.Clone()));
       scope.AddVariable(new Variable(scope.TranslateName("TargetVariable"), (IItem)GetVariable("TargetVariable").Value.Clone()));
+      scope.AddVariable(new Variable(scope.TranslateName("TrainingSamplesStart"), (IItem)GetVariable("TrainingSamplesStart").Value.Clone()));
+      scope.AddVariable(new Variable(scope.TranslateName("TrainingSamplesEnd"), (IItem)GetVariable("TrainingSamplesEnd").Value.Clone()));
+      scope.AddVariable(new Variable(scope.TranslateName("ValidationSamplesStart"), (IItem)GetVariable("ValidationSamplesStart").Value.Clone()));
+      scope.AddVariable(new Variable(scope.TranslateName("ValidationSamplesEnd"), (IItem)GetVariable("ValidationSamplesEnd").Value.Clone()));
       return null;
     }

trunk/sources/HeuristicLab.StructureIdentification/StructIdProblemInjectorView.Designer.cs

@@ -51 +51 @@
       this.variableInfosTabPage = new System.Windows.Forms.TabPage();
       this.operatorBaseVariableInfosView = new HeuristicLab.Core.OperatorBaseVariableInfosView();
+      this.tabPage1 = new System.Windows.Forms.TabPage();
+      this.operatorBaseVariablesView = new HeuristicLab.Core.OperatorBaseVariablesView();
       this.descriptionTabPage = new System.Windows.Forms.TabPage();
       this.operatorBaseDescriptionView = new HeuristicLab.Core.OperatorBaseDescriptionView();
@@ -57 +59 @@
       this.dataTabPage.SuspendLayout();
       this.variableInfosTabPage.SuspendLayout();
+      this.tabPage1.SuspendLayout();
       this.descriptionTabPage.SuspendLayout();
       this.SuspendLayout();
@@ -81 +84 @@
       this.tabControl.Controls.Add(this.dataTabPage);
       this.tabControl.Controls.Add(this.variableInfosTabPage);
+      this.tabControl.Controls.Add(this.tabPage1);
       this.tabControl.Controls.Add(this.descriptionTabPage);
       this.tabControl.Dock = System.Windows.Forms.DockStyle.Fill;
@@ -122 +126 @@
       this.operatorBaseVariableInfosView.TabIndex = 0;
       // 
+      // tabPage1
+      // 
+      this.tabPage1.Controls.Add(this.operatorBaseVariablesView);
+      this.tabPage1.Location = new System.Drawing.Point(4, 22);
+      this.tabPage1.Name = "tabPage1";
+      this.tabPage1.Padding = new System.Windows.Forms.Padding(3);
+      this.tabPage1.Size = new System.Drawing.Size(499, 425);
+      this.tabPage1.TabIndex = 3;
+      this.tabPage1.Text = "Variables";
+      this.tabPage1.UseVisualStyleBackColor = true;
+      // 
+      // operatorBaseVariablesView
+      // 
+      this.operatorBaseVariablesView.Caption = "Operator";
+      this.operatorBaseVariablesView.Dock = System.Windows.Forms.DockStyle.Fill;
+      this.operatorBaseVariablesView.Location = new System.Drawing.Point(3, 3);
+      this.operatorBaseVariablesView.Name = "operatorBaseVariablesView";
+      this.operatorBaseVariablesView.Operator = null;
+      this.operatorBaseVariablesView.Size = new System.Drawing.Size(493, 419);
+      this.operatorBaseVariablesView.TabIndex = 0;
+      // 
       // descriptionTabPage
       // 
@@ -143 +168 @@
       this.operatorBaseDescriptionView.TabIndex = 0;
       // 
-      // datasetView
+      // datasetView1
       // 
       this.datasetView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
                   | System.Windows.Forms.AnchorStyles.Left)
                   | System.Windows.Forms.AnchorStyles.Right)));
-      this.datasetView.Caption = "View";
+      this.datasetView.Caption = "Editor";
+      this.datasetView.Dataset = null;
+      this.datasetView.Filename = null;
       this.datasetView.Location = new System.Drawing.Point(6, 33);
-      this.datasetView.Name = "datasetView";
+      this.datasetView.Name = "datasetView1";
       this.datasetView.Size = new System.Drawing.Size(487, 386);
       this.datasetView.TabIndex = 7;
@@ -164 +191 @@
       this.dataTabPage.ResumeLayout(false);
       this.variableInfosTabPage.ResumeLayout(false);
+      this.tabPage1.ResumeLayout(false);
       this.descriptionTabPage.ResumeLayout(false);
       this.ResumeLayout(false);
@@ -179 +207 @@
     private System.Windows.Forms.TabPage descriptionTabPage;
     private HeuristicLab.Core.OperatorBaseDescriptionView operatorBaseDescriptionView;
+    private System.Windows.Forms.TabPage tabPage1;
+    private HeuristicLab.Core.OperatorBaseVariablesView operatorBaseVariablesView;
     private HeuristicLab.DataAnalysis.DatasetView datasetView;
   }

trunk/sources/HeuristicLab.StructureIdentification/StructIdProblemInjectorView.cs

@@ -89 +89 @@
         if (success) {
           Dataset dataset = (Dataset)StructIdProblemInjector.GetVariable("Dataset").Value;
-          dataset.Rows = parser.TrainingSamplesEnd - parser.TrainingSamplesStart;
+          dataset.Rows = parser.Rows;
           dataset.Columns = parser.Columns;
           dataset.VariableNames = parser.VariableNames;
@@ -95 +95 @@
           dataset.Samples = new double[dataset.Rows * dataset.Columns];
           Array.Copy(parser.Samples, dataset.Samples, dataset.Columns * dataset.Rows);
-
+          ((IntData)StructIdProblemInjector.GetVariable("TrainingSamplesStart").Value).Data = parser.TrainingSamplesStart;
+          ((IntData)StructIdProblemInjector.GetVariable("TrainingSamplesEnd").Value).Data = parser.TrainingSamplesEnd;
+          ((IntData)StructIdProblemInjector.GetVariable("ValidationSamplesStart").Value).Data = parser.ValidationSamplesStart;
+          ((IntData)StructIdProblemInjector.GetVariable("ValidationSamplesEnd").Value).Data = parser.ValidationSamplesEnd;
           ((IntData)StructIdProblemInjector.GetVariable("TargetVariable").Value).Data = parser.TargetVariable;
           Refresh();