[17479] | 1 | using HEAL.Attic;
|
---|
| 2 | using HeuristicLab.Algorithms.OESRALPS.Evaluators;
|
---|
| 3 | using HeuristicLab.Analysis;
|
---|
| 4 | using HeuristicLab.Common;
|
---|
| 5 | using HeuristicLab.Core;
|
---|
| 6 | using HeuristicLab.Data;
|
---|
| 7 | using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
|
---|
| 8 | using HeuristicLab.Optimization;
|
---|
| 9 | using HeuristicLab.Parameters;
|
---|
| 10 | using HeuristicLab.Problems.DataAnalysis;
|
---|
| 11 | using HeuristicLab.Problems.DataAnalysis.Symbolic;
|
---|
| 12 | using HeuristicLab.Random;
|
---|
| 13 | using System;
|
---|
| 14 | using System.Collections.Generic;
|
---|
| 15 | using System.Linq;
|
---|
| 16 | using System.Text;
|
---|
| 17 | using System.Threading.Tasks;
|
---|
| 18 |
|
---|
| 19 | namespace HeuristicLab.Algorithms.OESRALPS.Analyzers
|
---|
| 20 | {
|
---|
| 21 | [Item("SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer", "An operator that analyzes the validation best symbolic data analysis solution for single objective symbolic data analysis problems.")]
|
---|
| 22 | [StorableType("CD82C026-CF68-40D7-A898-77EA61972DE9")]
|
---|
| 23 | public abstract class SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer<S, T, U>
|
---|
| 24 | : SymbolicDataAnalysisSingleObjectiveLayerValidationAnalyzer<T, U>, IIterationBasedOperator
|
---|
| 25 | where S : class, ISymbolicDataAnalysisSolution
|
---|
| 26 | where T : class, ISymbolicDataAnalysisSingleObjectiveEvaluator<U>
|
---|
| 27 | where U : class, IDataAnalysisProblemData
|
---|
| 28 | {
|
---|
| 29 | private const string ValidationBestSolutionParameterName = "Best validation solution";
|
---|
| 30 | private const string ValidationBestSolutionQualityParameterName = "Best validation solution quality";
|
---|
| 31 | private const string ValidationBestSolutionGenerationParameterName = "Best validation solution generation";
|
---|
| 32 |
|
---|
| 33 | private const string TrainingBestSolutionParameterName = "Best training solution";
|
---|
| 34 | private const string TrainingBestSolutionQualityParameterName = "Best training solution quality";
|
---|
| 35 | private const string TrainingBestSolutionGenerationParameterName = "Best training solution generation";
|
---|
| 36 |
|
---|
| 37 | private const string UpdateAlwaysParameterName = "Always update best solution";
|
---|
| 38 | private const string IterationsParameterName = "Generations";
|
---|
| 39 | private const string MaximumIterationsParameterName = "Maximum Iterations";
|
---|
| 40 |
|
---|
| 41 | protected const string LayerResultsParameterName = "LayerResults";
|
---|
| 42 | protected const string EstimationLimitsParameterName = "EstimationLimits";
|
---|
| 43 |
|
---|
| 44 | protected const string LayerOfBestValidationSolutionParameterName = "Layer (Validation)";
|
---|
| 45 | protected const string LayerOfBestTrainingSolutionParameterName = "Layer (Training)";
|
---|
| 46 |
|
---|
| 47 | protected const string LayerOfBestSolutionDescription = "The layer in which the best solution has been found.";
|
---|
| 48 | private const string ValidationBestSolutionTableParameterName = "Best solution chart";
|
---|
| 49 |
|
---|
| 50 | private const string OpenLayersParameterName = "OpenLayers";
|
---|
| 51 |
|
---|
| 52 | private const string ErrorMeasureChart = "Error Measure of best solution Chart";
|
---|
| 53 |
|
---|
| 54 | private const string ValidationMAETestSetQuality = "Validation MAE (test)";
|
---|
| 55 | private const string ValidationMAETrainingSetQuality = "Validation MAE (training)";
|
---|
| 56 |
|
---|
| 57 | private const string ValidationMSETestSetQuality = "Validation MSE (test)";
|
---|
| 58 | private const string ValidationMSETrainingSetQuality = "Validation MSE (training)";
|
---|
| 59 |
|
---|
| 60 | private const string ValidationRMSETestSetQuality = "Validation RMSE (test)";
|
---|
| 61 | private const string ValidationRMSETrainingSetQuality = "Validation RMSE (training)";
|
---|
| 62 |
|
---|
| 63 | private const string ValidationPearsonR2TestSetQuality = "Validation PearsonR2 (test)";
|
---|
| 64 | private const string ValidationPearsonR2TrainingSetQuality = "Validation PearsonR2 (training)";
|
---|
| 65 |
|
---|
| 66 | private const string TrainingMAETestSetQuality = "Training MAE (test)";
|
---|
| 67 | private const string TrainingMAETrainingSetQuality = "Training MAE (training)";
|
---|
| 68 |
|
---|
| 69 | private const string TrainingMSETestSetQuality = "Training MSE (test)";
|
---|
| 70 | private const string TrainingMSETrainingSetQuality = "Training MSE (training)";
|
---|
| 71 |
|
---|
| 72 | private const string TrainingRMSETestSetQuality = "Training RMSE (test)";
|
---|
| 73 | private const string TrainingRMSETrainingSetQuality = "Training RMSE (training)";
|
---|
| 74 |
|
---|
| 75 | private const string TrainingPearsonR2TestSetQuality = "Training PearsonR2 (test)";
|
---|
| 76 | private const string TrainingPearsonR2TrainingSetQuality = "Training PearsonR2 (training)";
|
---|
| 77 |
|
---|
| 78 | private const string PreviousWindowBestValidationSolutionQuality = "Quality on previous window (Best Validation Solution)";
|
---|
| 79 | private const string NextWindowBestValidationSolutionQuality = "Quality on next window (Best Validation Solution)";
|
---|
| 80 |
|
---|
| 81 | private bool reevaluateValidationResult = false;
|
---|
| 82 |
|
---|
| 83 | #region parameter properties
|
---|
| 84 | public ILookupParameter<S> ValidationBestSolutionParameter {
|
---|
| 85 | get { return (ILookupParameter<S>)Parameters[ValidationBestSolutionParameterName]; }
|
---|
| 86 | }
|
---|
| 87 | public ILookupParameter<DoubleValue> ValidationBestSolutionQualityParameter {
|
---|
| 88 | get { return (ILookupParameter<DoubleValue>)Parameters[ValidationBestSolutionQualityParameterName]; }
|
---|
| 89 | }
|
---|
| 90 | public ILookupParameter<IntValue> ValidationBestSolutionGenerationParameter {
|
---|
| 91 | get { return (ILookupParameter<IntValue>)Parameters[ValidationBestSolutionGenerationParameterName]; }
|
---|
| 92 | }
|
---|
| 93 | public IFixedValueParameter<BoolValue> UpdateAlwaysParameter {
|
---|
| 94 | get { return (IFixedValueParameter<BoolValue>)Parameters[UpdateAlwaysParameterName]; }
|
---|
| 95 | }
|
---|
| 96 | public ILookupParameter<IntValue> IterationsParameter {
|
---|
| 97 | get { return (ILookupParameter<IntValue>)Parameters[IterationsParameterName]; }
|
---|
| 98 | }
|
---|
| 99 | public IValueLookupParameter<IntValue> MaximumIterationsParameter {
|
---|
| 100 | get { return (IValueLookupParameter<IntValue>)Parameters[MaximumIterationsParameterName]; }
|
---|
| 101 | }
|
---|
| 102 | public IScopeTreeLookupParameter<ResultCollection> LayerResultsParameterParameter {
|
---|
| 103 | get { return (IScopeTreeLookupParameter<ResultCollection>)Parameters[LayerResultsParameterName]; }
|
---|
| 104 | }
|
---|
| 105 | public IValueLookupParameter<DoubleLimit> EstimationLimitsParameter {
|
---|
| 106 | get { return (IValueLookupParameter<DoubleLimit>)Parameters[EstimationLimitsParameterName]; }
|
---|
| 107 | }
|
---|
| 108 | public ILookupParameter<IntValue> OpenLayersParameter {
|
---|
| 109 | get { return (ILookupParameter<IntValue>)Parameters[OpenLayersParameterName]; }
|
---|
| 110 | }
|
---|
| 111 | #endregion
|
---|
| 112 |
|
---|
| 113 | #region properties
|
---|
| 114 | public S ValidationBestSolution {
|
---|
| 115 | get { return ValidationBestSolutionParameter.ActualValue; }
|
---|
| 116 | set { ValidationBestSolutionParameter.ActualValue = value; }
|
---|
| 117 | }
|
---|
| 118 | public DoubleValue ValidationBestSolutionQuality {
|
---|
| 119 | get { return ValidationBestSolutionQualityParameter.ActualValue; }
|
---|
| 120 | set { ValidationBestSolutionQualityParameter.ActualValue = value; }
|
---|
| 121 | }
|
---|
| 122 | public BoolValue UpdateAlways {
|
---|
| 123 | get { return UpdateAlwaysParameter.Value; }
|
---|
| 124 | }
|
---|
| 125 | #endregion
|
---|
| 126 |
|
---|
| 127 | [StorableConstructor]
|
---|
| 128 | protected SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer(StorableConstructorFlag _) : base(_) { }
|
---|
| 129 | protected SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer(SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer<S, T, U> original, Cloner cloner) : base(original, cloner) { }
|
---|
| 130 | public SymbolicDataAnalysisSingleObjectiveValidationLayerBestSolutionAnalyzer()
|
---|
| 131 | : base()
|
---|
| 132 | {
|
---|
| 133 | Parameters.Add(new LookupParameter<S>(ValidationBestSolutionParameterName, "The validation best symbolic data analyis solution."));
|
---|
| 134 | Parameters.Add(new LookupParameter<DoubleValue>(ValidationBestSolutionQualityParameterName, "The quality of the validation best symbolic data analysis solution."));
|
---|
| 135 | Parameters.Add(new LookupParameter<IntValue>(ValidationBestSolutionGenerationParameterName, "The generation in which the best validation solution was found."));
|
---|
| 136 | Parameters.Add(new FixedValueParameter<BoolValue>(UpdateAlwaysParameterName, "Determines if the best validation solution should always be updated regardless of its quality.", new BoolValue(true)));
|
---|
| 137 | Parameters.Add(new LookupParameter<IntValue>(IterationsParameterName, "The number of performed iterations."));
|
---|
| 138 | Parameters.Add(new ValueLookupParameter<IntValue>(MaximumIterationsParameterName, "The maximum number of performed iterations.") { Hidden = true });
|
---|
| 139 |
|
---|
| 140 | Parameters.Add(new ValueLookupParameter<DoubleLimit>(EstimationLimitsParameterName, "The lower and upper limit for the estimated values produced by the symbolic regression model."));
|
---|
| 141 | Parameters.Add(new ScopeTreeLookupParameter<ResultCollection>(LayerResultsParameterName, "Results of all Layers.") { Depth = 1 });
|
---|
| 142 |
|
---|
| 143 | Parameters.Add(new LookupParameter<IntValue>(OpenLayersParameterName, "Open Layers"));
|
---|
| 144 |
|
---|
| 145 | UpdateAlwaysParameter.Hidden = true;
|
---|
| 146 | }
|
---|
| 147 |
|
---|
| 148 | protected abstract S CreateSolution(ISymbolicExpressionTree bestTree, double bestQuality);
|
---|
| 149 |
|
---|
| 150 | public override IOperation Apply()
|
---|
| 151 | {
|
---|
| 152 | IEnumerable<int> rows = GenerateRowsToEvaluate();
|
---|
| 153 | if (!rows.Any()) return base.Apply();
|
---|
| 154 |
|
---|
| 155 | #region find best validation tree
|
---|
| 156 | // Obtain result collections from layers
|
---|
| 157 | LayerResultsParameterParameter.Depth = 1;
|
---|
| 158 | var resultCollections = LayerResultsParameterParameter.ActualValue;
|
---|
| 159 | var bestLayerValidationSolution
|
---|
| 160 | = resultCollections
|
---|
| 161 | .Where(result =>
|
---|
| 162 | result.ContainsKey(ValidationBestSolutionParameterName) &&
|
---|
| 163 | result.ContainsKey(ValidationBestSolutionQualityParameterName) &&
|
---|
| 164 | result.ContainsKey(ValidationBestSolutionGenerationParameterName)
|
---|
| 165 | )
|
---|
| 166 | .Select((item, index) =>
|
---|
| 167 | new LayerSolution()
|
---|
| 168 | {
|
---|
| 169 | Solution = (S)item[ValidationBestSolutionParameterName].Value,
|
---|
| 170 | Quality = (DoubleValue)item[ValidationBestSolutionQualityParameterName].Value,
|
---|
| 171 | Generation = (IntValue)item[ValidationBestSolutionGenerationParameterName].Value,
|
---|
| 172 | Layer = new IntValue(index) // TODO Ask Affenzeller if layers should be zero indexed
|
---|
| 173 | }).ToArray();
|
---|
| 174 |
|
---|
| 175 | // TODO
|
---|
| 176 | // Check if TrainingBestSolutionAnalyzer added to LayerAnalyzer
|
---|
| 177 | // if false, add it and restart
|
---|
| 178 | // else this analyzer is at the wrong scope, must be set as Analyzer not Layer analyzer
|
---|
| 179 |
|
---|
| 180 | LayerSolution bestValidationSolution;
|
---|
| 181 | if (Maximization.Value)
|
---|
| 182 | bestValidationSolution = bestLayerValidationSolution.OrderByDescending(layerSolution => layerSolution.Quality.Value).First();
|
---|
| 183 | else
|
---|
| 184 | bestValidationSolution = bestLayerValidationSolution.OrderBy(layerSolution => layerSolution.Quality.Value).First();
|
---|
| 185 | #endregion
|
---|
| 186 |
|
---|
| 187 | #region find best training tree
|
---|
| 188 | LayerResultsParameterParameter.Depth = 1;
|
---|
| 189 | var bestLayerTrainingSolution
|
---|
| 190 | = resultCollections
|
---|
| 191 | .Where(result =>
|
---|
| 192 | result.ContainsKey(TrainingBestSolutionParameterName) &&
|
---|
| 193 | result.ContainsKey(TrainingBestSolutionQualityParameterName) &&
|
---|
| 194 | result.ContainsKey(TrainingBestSolutionGenerationParameterName)
|
---|
| 195 | )
|
---|
| 196 | .Select((item, index) =>
|
---|
| 197 | new LayerSolution()
|
---|
| 198 | {
|
---|
| 199 | Solution = (S)item[TrainingBestSolutionParameterName].Value,
|
---|
| 200 | Quality = (DoubleValue)item[TrainingBestSolutionQualityParameterName].Value,
|
---|
| 201 | Generation = (IntValue)item[TrainingBestSolutionGenerationParameterName].Value,
|
---|
| 202 | Layer = new IntValue(index) // TODO Ask Affenzeller if layers should be zero indexed
|
---|
| 203 | }).ToArray();
|
---|
| 204 |
|
---|
| 205 | // TODO
|
---|
| 206 | // Check if TrainingBestSolutionAnalyzer added to LayerAnalyzer
|
---|
| 207 | // if false, add it and restart
|
---|
| 208 | // else this analyzer is at the wrong scope, must be set as Analyzer not Layer analyzer
|
---|
| 209 |
|
---|
| 210 | LayerSolution bestTrainingSolution;
|
---|
| 211 | if (Maximization.Value)
|
---|
| 212 | bestTrainingSolution = bestLayerTrainingSolution.OrderByDescending(layerSolution => layerSolution.Quality.Value).First();
|
---|
| 213 | else
|
---|
| 214 | bestTrainingSolution = bestLayerTrainingSolution.OrderBy(layerSolution => layerSolution.Quality.Value).First();
|
---|
| 215 | #endregion
|
---|
| 216 |
|
---|
| 217 | var results = ResultCollection;
|
---|
| 218 | #region Add Parameter
|
---|
| 219 | if (!results.ContainsKey(ValidationBestSolutionParameterName))
|
---|
| 220 | {
|
---|
| 221 | results.Add(new Result(ValidationBestSolutionParameterName, ValidationBestSolutionParameter.Description, typeof(S)));
|
---|
| 222 | ValidationPartitionParameter.ActualValue.ValueChanged += ValidationPartition_ValueChanged;
|
---|
| 223 | }
|
---|
| 224 | if (!results.ContainsKey(ValidationBestSolutionQualityParameterName))
|
---|
| 225 | results.Add(new Result(ValidationBestSolutionQualityParameterName, ValidationBestSolutionQualityParameter.Description, typeof(DoubleValue)));
|
---|
| 226 | if (!results.ContainsKey(ValidationBestSolutionGenerationParameterName))
|
---|
| 227 | results.Add(new Result(ValidationBestSolutionGenerationParameterName, ValidationBestSolutionGenerationParameter.Description, typeof(IntValue)));
|
---|
| 228 | if (!results.ContainsKey(LayerOfBestValidationSolutionParameterName))
|
---|
| 229 | results.Add(new Result(LayerOfBestValidationSolutionParameterName, LayerOfBestSolutionDescription, typeof(IntValue)));
|
---|
| 230 | if (!results.ContainsKey(ValidationBestSolutionTableParameterName))
|
---|
| 231 | results.Add(new Result(ValidationBestSolutionTableParameterName, new DataTable(ValidationBestSolutionTableParameterName)));
|
---|
| 232 |
|
---|
| 233 | var validationBestSolutionTable = (DataTable)results[ValidationBestSolutionTableParameterName].Value;
|
---|
| 234 |
|
---|
| 235 | if (!validationBestSolutionTable.Rows.ContainsKey(ValidationBestSolutionQualityParameterName))
|
---|
| 236 | validationBestSolutionTable.Rows.Add(new DataRow(ValidationBestSolutionQualityParameterName));
|
---|
| 237 | if (!validationBestSolutionTable.Rows.ContainsKey(LayerOfBestValidationSolutionParameterName))
|
---|
| 238 | validationBestSolutionTable.Rows.Add(new DataRow(LayerOfBestValidationSolutionParameterName) { VisualProperties = { SecondYAxis = true } });
|
---|
| 239 | if (!validationBestSolutionTable.Rows.ContainsKey(OpenLayersParameterName))
|
---|
| 240 | validationBestSolutionTable.Rows.Add(new DataRow(OpenLayersParameterName) { VisualProperties = { SecondYAxis = true } });
|
---|
| 241 |
|
---|
| 242 | if (!validationBestSolutionTable.Rows.ContainsKey(TrainingBestSolutionQualityParameterName))
|
---|
| 243 | validationBestSolutionTable.Rows.Add(new DataRow(TrainingBestSolutionQualityParameterName));
|
---|
| 244 | if (!validationBestSolutionTable.Rows.ContainsKey(LayerOfBestTrainingSolutionParameterName))
|
---|
| 245 | validationBestSolutionTable.Rows.Add(new DataRow(LayerOfBestTrainingSolutionParameterName) { VisualProperties = { SecondYAxis = true } });
|
---|
| 246 |
|
---|
| 247 | if (!validationBestSolutionTable.Rows.ContainsKey(PreviousWindowBestValidationSolutionQuality))
|
---|
| 248 | validationBestSolutionTable.Rows.Add(new DataRow(PreviousWindowBestValidationSolutionQuality));
|
---|
| 249 | if (!validationBestSolutionTable.Rows.ContainsKey(NextWindowBestValidationSolutionQuality))
|
---|
| 250 | validationBestSolutionTable.Rows.Add(new DataRow(NextWindowBestValidationSolutionQuality));
|
---|
| 251 |
|
---|
| 252 | if (!results.ContainsKey(ErrorMeasureChart))
|
---|
| 253 | results.Add(new Result(ErrorMeasureChart, new DataTable(ErrorMeasureChart) { VisualProperties = { YAxisLogScale = true } }));
|
---|
| 254 |
|
---|
| 255 | var MSEBestSolutionErrorTable = (DataTable)results[ErrorMeasureChart].Value;
|
---|
| 256 |
|
---|
| 257 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationMAETestSetQuality))
|
---|
| 258 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationMAETestSetQuality));
|
---|
| 259 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationMAETrainingSetQuality))
|
---|
| 260 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationMAETrainingSetQuality));
|
---|
| 261 |
|
---|
| 262 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationMSETestSetQuality))
|
---|
| 263 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationMSETestSetQuality));
|
---|
| 264 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationMSETrainingSetQuality))
|
---|
| 265 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationMSETrainingSetQuality));
|
---|
| 266 |
|
---|
| 267 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationRMSETestSetQuality))
|
---|
| 268 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationRMSETestSetQuality));
|
---|
| 269 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationRMSETrainingSetQuality))
|
---|
| 270 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationRMSETrainingSetQuality));
|
---|
| 271 |
|
---|
| 272 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationPearsonR2TestSetQuality))
|
---|
| 273 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationPearsonR2TestSetQuality) { VisualProperties = { SecondYAxis = true } });
|
---|
| 274 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(ValidationPearsonR2TrainingSetQuality))
|
---|
| 275 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(ValidationPearsonR2TrainingSetQuality) { VisualProperties = { SecondYAxis = true } });
|
---|
| 276 |
|
---|
| 277 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingMSETestSetQuality))
|
---|
| 278 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingMSETestSetQuality));
|
---|
| 279 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingMSETrainingSetQuality))
|
---|
| 280 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingMSETrainingSetQuality));
|
---|
| 281 |
|
---|
| 282 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingRMSETestSetQuality))
|
---|
| 283 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingRMSETestSetQuality));
|
---|
| 284 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingRMSETrainingSetQuality))
|
---|
| 285 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingRMSETrainingSetQuality));
|
---|
| 286 |
|
---|
| 287 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingMAETestSetQuality))
|
---|
| 288 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingMAETestSetQuality));
|
---|
| 289 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingMAETrainingSetQuality))
|
---|
| 290 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingMAETrainingSetQuality));
|
---|
| 291 |
|
---|
| 292 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingPearsonR2TestSetQuality))
|
---|
| 293 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingPearsonR2TestSetQuality) { VisualProperties = { SecondYAxis = true } });
|
---|
| 294 | if (!MSEBestSolutionErrorTable.Rows.ContainsKey(TrainingPearsonR2TrainingSetQuality))
|
---|
| 295 | MSEBestSolutionErrorTable.Rows.Add(new DataRow(TrainingPearsonR2TrainingSetQuality) { VisualProperties = { SecondYAxis = true } });
|
---|
| 296 |
|
---|
| 297 | #endregion
|
---|
| 298 |
|
---|
| 299 | #region Reevaluate current best solution
|
---|
| 300 | if (reevaluateValidationResult
|
---|
| 301 | && ValidationBestSolutionQuality != null
|
---|
| 302 | && ValidationPartitionParameter.ActualValue != null)
|
---|
| 303 | {
|
---|
| 304 | var evaluator = EvaluatorParameter.ActualValue;
|
---|
| 305 | var problemData = ProblemDataParameter.ActualValue;
|
---|
| 306 |
|
---|
| 307 | IExecutionContext childContext = (IExecutionContext)ExecutionContext.CreateChildOperation(evaluator);
|
---|
| 308 | var currentValidationBestSolution = (S)results[ValidationBestSolutionParameterName].Value;
|
---|
| 309 | ValidationBestSolutionQuality.Value =
|
---|
| 310 | evaluator.Evaluate(
|
---|
| 311 | childContext,
|
---|
| 312 | currentValidationBestSolution.Model.SymbolicExpressionTree,
|
---|
| 313 | problemData,
|
---|
| 314 | Enumerable.Range(
|
---|
| 315 | ValidationPartitionParameter.ActualValue.Start,
|
---|
| 316 | ValidationPartitionParameter.ActualValue.End - ValidationPartitionParameter.ActualValue.Start
|
---|
| 317 | ));
|
---|
| 318 | results[ValidationBestSolutionQualityParameterName].Value = ValidationBestSolutionQuality;
|
---|
| 319 | results[ValidationBestSolutionGenerationParameterName].Value = IterationsParameter.ActualValue;
|
---|
| 320 | reevaluateValidationResult = false;
|
---|
| 321 | }
|
---|
| 322 | #endregion
|
---|
| 323 |
|
---|
| 324 | if (bestValidationSolution.Solution != null && (UpdateAlways.Value || ValidationBestSolutionQuality == null ||
|
---|
| 325 | IsBetter(bestValidationSolution.Quality.Value, ValidationBestSolutionQuality.Value, Maximization.Value)))
|
---|
| 326 | {
|
---|
| 327 | ValidationBestSolution = CreateSolution(bestValidationSolution.Solution.Model.SymbolicExpressionTree, bestValidationSolution.Quality.Value);
|
---|
| 328 | ValidationBestSolutionQuality = bestValidationSolution.Quality;
|
---|
| 329 |
|
---|
| 330 | results[ValidationBestSolutionParameter.Name].Value = ValidationBestSolution;
|
---|
| 331 | results[ValidationBestSolutionQualityParameter.Name].Value = ValidationBestSolutionQuality;
|
---|
| 332 | // TODO Ask Affenzeller if current iteration should be shown or the iteration when the solution was created
|
---|
| 333 | results[ValidationBestSolutionGenerationParameter.Name].Value = bestValidationSolution.Generation;
|
---|
| 334 | results[LayerOfBestValidationSolutionParameterName].Value = bestValidationSolution.Layer;
|
---|
| 335 | }
|
---|
| 336 |
|
---|
| 337 | #region create charts
|
---|
| 338 | validationBestSolutionTable.Rows[OpenLayersParameterName].Values.Add(OpenLayersParameter.ActualValue.Value);
|
---|
| 339 | validationBestSolutionTable.Rows[ValidationBestSolutionQualityParameterName].Values.Add(ValidationBestSolutionQualityParameter.ActualValue.Value);
|
---|
| 340 | validationBestSolutionTable.Rows[LayerOfBestValidationSolutionParameterName].Values.Add(((IntValue)results[LayerOfBestValidationSolutionParameterName].Value).Value + 1);
|
---|
| 341 | validationBestSolutionTable.Rows[TrainingBestSolutionQualityParameterName].Values.Add(bestTrainingSolution.Quality.Value);
|
---|
| 342 | validationBestSolutionTable.Rows[LayerOfBestTrainingSolutionParameterName].Values.Add(bestTrainingSolution.Layer.Value + 1);
|
---|
| 343 |
|
---|
| 344 | results[ValidationBestSolutionTableParameterName].Value = validationBestSolutionTable;
|
---|
| 345 |
|
---|
| 346 | var bestValidationRegressionSolution = ValidationBestSolution as IRegressionSolution;
|
---|
| 347 | var bestTrainingRegressionSolution = bestTrainingSolution.Solution as IRegressionSolution;
|
---|
| 348 |
|
---|
| 349 | MSEBestSolutionErrorTable.Rows[ValidationMSETestSetQuality].Values.Add(bestValidationRegressionSolution.TestMeanSquaredError);
|
---|
| 350 | MSEBestSolutionErrorTable.Rows[ValidationMSETrainingSetQuality].Values.Add(bestValidationRegressionSolution.TrainingMeanSquaredError);
|
---|
| 351 |
|
---|
| 352 | MSEBestSolutionErrorTable.Rows[ValidationRMSETestSetQuality].Values.Add(bestValidationRegressionSolution.TestRootMeanSquaredError);
|
---|
| 353 | MSEBestSolutionErrorTable.Rows[ValidationRMSETrainingSetQuality].Values.Add(bestValidationRegressionSolution.TrainingRootMeanSquaredError);
|
---|
| 354 |
|
---|
| 355 | MSEBestSolutionErrorTable.Rows[ValidationMAETestSetQuality].Values.Add(bestValidationRegressionSolution.TestMeanAbsoluteError);
|
---|
| 356 | MSEBestSolutionErrorTable.Rows[ValidationMAETrainingSetQuality].Values.Add(bestValidationRegressionSolution.TrainingMeanAbsoluteError);
|
---|
| 357 |
|
---|
| 358 | MSEBestSolutionErrorTable.Rows[ValidationPearsonR2TestSetQuality].Values.Add(bestValidationRegressionSolution.TestRSquared);
|
---|
| 359 | MSEBestSolutionErrorTable.Rows[ValidationPearsonR2TrainingSetQuality].Values.Add(bestValidationRegressionSolution.TrainingRSquared);
|
---|
| 360 |
|
---|
| 361 | MSEBestSolutionErrorTable.Rows[TrainingMSETestSetQuality].Values.Add(bestTrainingRegressionSolution.TestMeanSquaredError);
|
---|
| 362 | MSEBestSolutionErrorTable.Rows[TrainingMSETrainingSetQuality].Values.Add(bestTrainingRegressionSolution.TrainingMeanSquaredError);
|
---|
| 363 |
|
---|
| 364 | MSEBestSolutionErrorTable.Rows[TrainingRMSETestSetQuality].Values.Add(bestTrainingRegressionSolution.TestRootMeanSquaredError);
|
---|
| 365 | MSEBestSolutionErrorTable.Rows[TrainingRMSETrainingSetQuality].Values.Add(bestTrainingRegressionSolution.TrainingRootMeanSquaredError);
|
---|
| 366 |
|
---|
| 367 | MSEBestSolutionErrorTable.Rows[TrainingMAETestSetQuality].Values.Add(bestTrainingRegressionSolution.TestMeanAbsoluteError);
|
---|
| 368 | MSEBestSolutionErrorTable.Rows[TrainingMAETrainingSetQuality].Values.Add(bestTrainingRegressionSolution.TrainingMeanAbsoluteError);
|
---|
| 369 |
|
---|
| 370 | MSEBestSolutionErrorTable.Rows[TrainingPearsonR2TestSetQuality].Values.Add(bestTrainingRegressionSolution.TestRSquared);
|
---|
| 371 | MSEBestSolutionErrorTable.Rows[TrainingPearsonR2TrainingSetQuality].Values.Add(bestTrainingRegressionSolution.TrainingRSquared);
|
---|
| 372 |
|
---|
| 373 | results[ErrorMeasureChart].Value = MSEBestSolutionErrorTable;
|
---|
| 374 | #endregion
|
---|
| 375 |
|
---|
| 376 | return base.Apply();
|
---|
| 377 | }
|
---|
| 378 |
|
---|
| 379 | private void ValidationPartition_ValueChanged(object sender, EventArgs e)
|
---|
| 380 | {
|
---|
| 381 | reevaluateValidationResult = true;
|
---|
| 382 | }
|
---|
| 383 |
|
---|
| 384 | private bool IsBetter(double lhs, double rhs, bool maximization)
|
---|
| 385 | {
|
---|
| 386 | if (maximization) return lhs > rhs;
|
---|
| 387 | else return lhs < rhs;
|
---|
| 388 | }
|
---|
| 389 |
|
---|
| 390 | protected override IEnumerable<int> GenerateRowsToEvaluate()
|
---|
| 391 | {
|
---|
| 392 | if (ValidationPartitionParameter.ActualValue == null
|
---|
| 393 | || TestPartitionParameter.ActualValue == null)
|
---|
| 394 | return base.GenerateRowsToEvaluate();
|
---|
| 395 |
|
---|
| 396 | int seed = RandomParameter.ActualValue.Next();
|
---|
| 397 | int samplesStart = ValidationPartitionParameter.ActualValue.Start;
|
---|
| 398 | int samplesEnd = ValidationPartitionParameter.ActualValue.End;
|
---|
| 399 | int testPartitionStart = TestPartitionParameter.ActualValue.Start;
|
---|
| 400 | int testPartitionEnd = TestPartitionParameter.ActualValue.End;
|
---|
| 401 |
|
---|
| 402 | if (samplesEnd < samplesStart) throw new ArgumentException("Start value is larger than end value.");
|
---|
| 403 | int count = (int)((samplesEnd - samplesStart) * RelativeNumberOfEvaluatedSamplesParameter.ActualValue.Value);
|
---|
| 404 | if (count == 0) count = 1;
|
---|
| 405 | return RandomEnumerable.SampleRandomNumbers(seed, samplesStart, samplesEnd, count)
|
---|
| 406 | .Where(i => i < testPartitionStart && i < ProblemDataParameter.ActualValue.Dataset.Rows);
|
---|
| 407 | }
|
---|
| 408 |
|
---|
| 409 | [StorableType("CD82C026-CF61-40D7-A898-77EA61992DE9")]
|
---|
| 410 | class LayerSolution
|
---|
| 411 | {
|
---|
| 412 | [Storable]
|
---|
| 413 | public S Solution { get; set; }
|
---|
| 414 | [Storable]
|
---|
| 415 | public DoubleValue Quality { get; set; }
|
---|
| 416 | [Storable]
|
---|
| 417 | public IntValue Generation { get; set; }
|
---|
| 418 | [Storable]
|
---|
| 419 | public IntValue Layer { get; set; }
|
---|
| 420 | }
|
---|
| 421 | }
|
---|
| 422 | }
|
---|