Changeset 16051
 Timestamp:
 08/06/18 16:16:51 (3 years ago)
 Location:
 branches/2904_CalculateImpacts
 Files:

 2 edited
Legend:
 Unmodified
 Added
 Removed

branches/2904_CalculateImpacts/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs
r16041 r16051 90 90 public RegressionSolutionVariableImpactsCalculator() 91 91 : base() { 92 Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum. Median)));92 Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Shuffle))); 93 93 Parameters.Add(new FixedValueParameter<EnumValue<FactorReplacementMethodEnum>>(FactorReplacementParameterName, "The replacement method for factor variables during impact calculation.", new EnumValue<FactorReplacementMethodEnum>(FactorReplacementMethodEnum.Best))); 94 94 Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training))); … … 110 110 FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best, 111 111 DataPartitionEnum dataPartition = DataPartitionEnum.Training) { 112 IEnumerable<int> rows = (GetPartitionRows(dataPartition, solution.ProblemData)); 112 113 IEnumerable<int> rows = GetPartitionRows(dataPartition, solution.ProblemData); 113 114 IEnumerable<double> estimatedValues = solution.GetEstimatedValues(rows); 114 115 return CalculateImpacts(solution.Model, solution.ProblemData, estimatedValues, rows, replacementMethod, factorReplacementMethod); … … 122 123 ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle, 123 124 FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) { 124 //Calculate original qualityvalues (via calculator, default is R²) 125 OnlineCalculatorError error; 125 126 //fholzing: try and catch in case a different dataset is loaded, otherwise statement is neglectable 127 var missingVariables = model.VariablesUsedForPrediction.Except(problemData.Dataset.VariableNames); 128 if (missingVariables.Any()) { 129 throw new InvalidOperationException(string.Format("Can not calculate variable impacts, because the model uses inputs missing in the dataset ({0})", string.Join(", ", missingVariables))); 130 } 126 131 IEnumerable<double> targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows); 127 var originalCalculatorValue = CalculateVariableImpact(targetValues, estimatedValues, out error); 128 if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation."); 132 var originalQuality = CalculateQuality(targetValues, estimatedValues); 129 133 130 134 var impacts = new Dictionary<string, double>(); 131 135 var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(model.VariablesUsedForPrediction)); 132 var allowedInputVariables = problemData.Dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();133 136 var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable(); 134 137 135 136 foreach (var inputVariable in allowedInputVariables) { 137 if (model.VariablesUsedForPrediction.Contains(inputVariable)) { 138 impacts[inputVariable] = CalculateImpact(inputVariable, model, modifiableDataset, rows, targetValues, originalCalculatorValue, replacementMethod, factorReplacementMethod); 139 } else { 140 impacts[inputVariable] = 0; 141 } 142 } 143 144 return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value)); 138 foreach (var inputVariable in inputvariables) { 139 impacts[inputVariable] = CalculateImpact(inputVariable, model, problemData, modifiableDataset, rows, replacementMethod, factorReplacementMethod, targetValues, originalQuality); 140 } 141 142 return impacts.Select(i => Tuple.Create(i.Key, i.Value)); 145 143 } 146 144 147 145 public static double CalculateImpact(string variableName, 148 146 IRegressionModel model, 147 IRegressionProblemData problemData, 149 148 ModifiableDataset modifiableDataset, 150 149 IEnumerable<int> rows, 150 ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle, 151 FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best, 152 IEnumerable<double> targetValues = null, 153 double quality = double.NaN) { 154 155 if (!model.VariablesUsedForPrediction.Contains(variableName)) { return 0.0; } 156 if (!problemData.Dataset.VariableNames.Contains(variableName)) { 157 throw new InvalidOperationException(string.Format("Can not calculate variable impact, because the model uses inputs missing in the dataset ({0})", variableName)); 158 } 159 160 if (targetValues == null) { 161 targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows); 162 } 163 if (quality == double.NaN) { 164 quality = CalculateQuality(model.GetEstimatedValues(modifiableDataset, rows), targetValues); 165 } 166 167 IList originalValues = null; 168 IList replacementValues = GetReplacementValues(modifiableDataset, variableName, model, rows, targetValues, out originalValues, replacementMethod, factorReplacementMethod); 169 170 double newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, replacementValues, targetValues); 171 double impact = quality  newValue; 172 173 return impact; 174 } 175 176 private static IList GetReplacementValues(ModifiableDataset modifiableDataset, 177 string variableName, 178 IRegressionModel model, 179 IEnumerable<int> rows, 151 180 IEnumerable<double> targetValues, 152 double originalValue,181 out IList originalValues, 153 182 ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle, 154 183 FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) { 155 double impact = 0; 156 OnlineCalculatorError error; 157 IRandom random; 158 double replacementValue; 159 IEnumerable<double> newEstimates = null; 160 double newValue = 0; 161 184 185 IList replacementValues = null; 162 186 if (modifiableDataset.VariableHasType<double>(variableName)) { 163 #region NumericalVariable 164 var originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList(); 165 List<double> replacementValues; 166 167 switch (replacementMethod) { 168 case ReplacementMethodEnum.Median: 169 replacementValue = rows.Select(r => originalValues[r]).Median(); 170 replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList(); 171 break; 172 case ReplacementMethodEnum.Average: 173 replacementValue = rows.Select(r => originalValues[r]).Average(); 174 replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList(); 175 break; 176 case ReplacementMethodEnum.Shuffle: 177 // new var has same empirical distribution but the relation to y is broken 178 random = new FastRandom(31415); 179 // prepare a complete column for the dataset 180 replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList(); 181 // shuffle only the selected rows 182 var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList(); 183 int i = 0; 184 // update column values 185 foreach (var r in rows) { 186 replacementValues[r] = shuffledValues[i++]; 187 } 188 break; 189 case ReplacementMethodEnum.Noise: 190 var avg = rows.Select(r => originalValues[r]).Average(); 191 var stdDev = rows.Select(r => originalValues[r]).StandardDeviation(); 192 random = new FastRandom(31415); 193 // prepare a complete column for the dataset 194 replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList(); 195 // update column values 196 foreach (var r in rows) { 197 replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev); 198 } 199 break; 200 201 default: 202 throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod)); 203 } 204 205 newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues); 206 newValue = CalculateVariableImpact(targetValues, newEstimates, out error); 207 if (error != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); } 208 209 impact = originalValue  newValue; 210 #endregion 187 originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList(); 188 replacementValues = GetReplacementValuesForDouble(modifiableDataset, rows, (List<double>)originalValues, replacementMethod); 211 189 } else if (modifiableDataset.VariableHasType<string>(variableName)) { 212 #region FactorVariable 213 var originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList(); 214 List<string> replacementValues; 215 216 switch (factorReplacementMethod) { 217 case FactorReplacementMethodEnum.Best: 218 // try replacing with all possible values and find the best replacement value 219 var smallestImpact = double.PositiveInfinity; 220 foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) { 221 newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, Enumerable.Repeat(repl, modifiableDataset.Rows).ToList()); 222 newValue = CalculateVariableImpact(targetValues, newEstimates, out error); 223 if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs."); 224 225 var curImpact = originalValue  newValue; 226 if (curImpact < smallestImpact) smallestImpact = curImpact; 227 } 228 impact = smallestImpact; 229 break; 230 case FactorReplacementMethodEnum.Mode: 231 var mostCommonValue = rows.Select(r => originalValues[r]) 232 .GroupBy(v => v) 233 .OrderByDescending(g => g.Count()) 234 .First().Key; 235 replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList(); 236 237 newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues); 238 newValue = CalculateVariableImpact(targetValues, newEstimates, out error); 239 if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs."); 240 241 impact = originalValue  newValue; 242 break; 243 case FactorReplacementMethodEnum.Shuffle: 244 // new var has same empirical distribution but the relation to y is broken 245 random = new FastRandom(31415); 246 // prepare a complete column for the dataset 247 replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList(); 248 // shuffle only the selected rows 249 var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList(); 250 int i = 0; 251 // update column values 252 foreach (var r in rows) { 253 replacementValues[r] = shuffledValues[i++]; 254 } 255 256 newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues); 257 newValue = CalculateVariableImpact(targetValues, newEstimates, out error); 258 if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs."); 259 260 impact = originalValue  newValue; 261 break; 262 default: 263 throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod)); 264 } 265 #endregion 190 originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList(); 191 replacementValues = GetReplacementValuesForString(model, modifiableDataset, variableName, rows, originalValues, targetValues, factorReplacementMethod); 266 192 } else { 267 193 throw new NotSupportedException("Variable not supported"); 268 194 } 269 195 270 return impact; 271 } 272 273 /// <summary> 274 /// Replaces the values of the original modelvariables with the replacement variables, calculates the new estimated values 275 /// and changes the value of the modelvariables back to the original ones. 276 /// </summary> 277 /// <param name="originalValues"></param> 278 /// <param name="model"></param> 279 /// <param name="variableName"></param> 280 /// <param name="modifiableDataset"></param> 281 /// <param name="rows"></param> 282 /// <param name="replacementValues"></param> 283 /// <returns></returns> 284 private static IEnumerable<double> GetReplacedEstimates( 196 return replacementValues; 197 } 198 199 private static IList GetReplacementValuesForDouble(ModifiableDataset modifiableDataset, 200 IEnumerable<int> rows, 201 List<double> originalValues, 202 ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle) { 203 204 IRandom random = new FastRandom(31415); 205 List<double> replacementValues; 206 double replacementValue; 207 208 switch (replacementMethod) { 209 case ReplacementMethodEnum.Median: 210 replacementValue = rows.Select(r => originalValues[r]).Median(); 211 replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList(); 212 break; 213 case ReplacementMethodEnum.Average: 214 replacementValue = rows.Select(r => originalValues[r]).Average(); 215 replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList(); 216 break; 217 case ReplacementMethodEnum.Shuffle: 218 // new var has same empirical distribution but the relation to y is broken 219 // prepare a complete column for the dataset 220 replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList(); 221 // shuffle only the selected rows 222 var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList(); 223 int i = 0; 224 // update column values 225 foreach (var r in rows) { 226 replacementValues[r] = shuffledValues[i++]; 227 } 228 break; 229 case ReplacementMethodEnum.Noise: 230 var avg = rows.Select(r => originalValues[r]).Average(); 231 var stdDev = rows.Select(r => originalValues[r]).StandardDeviation(); 232 // prepare a complete column for the dataset 233 replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList(); 234 // update column values 235 foreach (var r in rows) { 236 replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev); 237 } 238 break; 239 240 default: 241 throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod)); 242 } 243 244 return replacementValues; 245 } 246 247 private static IList GetReplacementValuesForString(IRegressionModel model, 248 ModifiableDataset modifiableDataset, 249 string variableName, 250 IEnumerable<int> rows, 285 251 IList originalValues, 252 IEnumerable<double> targetValues, 253 FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Shuffle) { 254 255 IList replacementValues = null; 256 IRandom random = new FastRandom(31415); 257 258 switch (factorReplacementMethod) { 259 case FactorReplacementMethodEnum.Best: 260 // try replacing with all possible values and find the best replacement value 261 var bestQuality = double.NegativeInfinity; 262 foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) { 263 List<string> curReplacementValues = Enumerable.Repeat(repl, modifiableDataset.Rows).ToList(); 264 //fholzing: this result could be used later on (theoretically), but is neglected for better readability/method consistency 265 var newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, replacementValues, targetValues); 266 var curQuality = newValue; 267 268 if (curQuality > bestQuality) { 269 bestQuality = curQuality; 270 replacementValues = curReplacementValues; 271 } 272 } 273 break; 274 case FactorReplacementMethodEnum.Mode: 275 var mostCommonValue = rows.Select(r => originalValues[r]) 276 .GroupBy(v => v) 277 .OrderByDescending(g => g.Count()) 278 .First().Key; 279 replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList(); 280 break; 281 case FactorReplacementMethodEnum.Shuffle: 282 // new var has same empirical distribution but the relation to y is broken 283 // prepare a complete column for the dataset 284 replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList(); 285 // shuffle only the selected rows 286 var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList(); 287 int i = 0; 288 // update column values 289 foreach (var r in rows) { 290 replacementValues[r] = shuffledValues[i++]; 291 } 292 break; 293 default: 294 throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod)); 295 } 296 297 return replacementValues; 298 } 299 300 private static double CalculateQualityForReplacement( 286 301 IRegressionModel model, 302 ModifiableDataset modifiableDataset, 287 303 string variableName, 288 ModifiableDataset modifiableDataset, 289 IEnumerable<int> rows, 290 IList replacementValues) { 304 IList originalValues, 305 IEnumerable<int> rows, 306 IList replacementValues, 307 IEnumerable<double> targetValues) { 308 291 309 modifiableDataset.ReplaceVariable(variableName, replacementValues); 292 310 //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements 293 311 var estimates = model.GetEstimatedValues(modifiableDataset, rows).ToList(); 312 var ret = CalculateQuality(targetValues, estimates); 294 313 modifiableDataset.ReplaceVariable(variableName, originalValues); 295 314 296 return estimates; 297 } 298 299 /// <summary> 300 /// Calculates and returns the VariableImpact (calculated via Pearsons R²). 301 /// </summary> 302 /// <param name="targetValues">The actual values</param> 303 /// <param name="estimatedValues">The calculated/replaced values</param> 304 /// <param name="errorState"></param> 305 /// <returns></returns> 306 public static double CalculateVariableImpact(IEnumerable<double> targetValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) { 307 //Theoretically, all calculators implement a static CalculateMethod which provides the same functionality 308 //as the code below does. But this way we can easily swap the calculator later on, so the user 309 //could choose a Calculator during runtime in future versions. 310 IOnlineCalculator calculator = new OnlinePearsonsRSquaredCalculator(); 311 IEnumerator<double> firstEnumerator = targetValues.GetEnumerator(); 312 IEnumerator<double> secondEnumerator = estimatedValues.GetEnumerator(); 313 314 // always move forward both enumerators (do not use shortcircuit evaluation!) 315 while (firstEnumerator.MoveNext() & secondEnumerator.MoveNext()) { 316 double original = firstEnumerator.Current; 317 double estimated = secondEnumerator.Current; 318 calculator.Add(original, estimated); 319 if (calculator.ErrorState != OnlineCalculatorError.None) break; 320 } 321 322 // check if both enumerators are at the end to make sure both enumerations have the same length 323 if (calculator.ErrorState == OnlineCalculatorError.None && 324 (secondEnumerator.MoveNext()  firstEnumerator.MoveNext())) { 325 throw new ArgumentException("Number of elements in first and second enumeration doesn't match."); 326 } else { 327 errorState = calculator.ErrorState; 328 return calculator.Value; 329 } 330 } 331 332 /// <summary> 333 /// Returns a collection of the rowindices for a given DataPartition (training or test) 334 /// </summary> 335 /// <param name="dataPartition"></param> 336 /// <param name="problemData"></param> 337 /// <returns></returns> 315 return ret; 316 } 317 318 public static double CalculateQuality(IEnumerable<double> targetValues, IEnumerable<double> estimatedValues) { 319 OnlineCalculatorError errorState; 320 var ret = OnlinePearsonsRCalculator.Calculate(targetValues, estimatedValues, out errorState); 321 if (errorState != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); } 322 return ret * ret; 323 } 324 338 325 public static IEnumerable<int> GetPartitionRows(DataPartitionEnum dataPartition, IRegressionProblemData problemData) { 339 326 IEnumerable<int> rows; 
branches/2904_CalculateImpacts/HeuristicLab.Problems.DataAnalysis.Views/3.4/Regression/RegressionSolutionVariableImpactsView.cs
r16042 r16051 141 141 142 142 rawVariableImpacts.Clear(); 143 originalVariableOrdering.ForEach(v => rawVariableImpacts.Add(new Tuple<string, double>(v, impacts.First(vv => vv.Item1 == v).Item2)));143 rawVariableImpacts.AddRange(impacts); 144 144 UpdateOrdering(); 145 145 } … … 164 164 165 165 //Calculate original qualityvalues (via calculator, default is R²) 166 OnlineCalculatorError error;167 166 IEnumerable<double> targetValuesPartition = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows); 168 167 IEnumerable<double> estimatedValuesPartition = Content.GetEstimatedValues(rows); 169 168 170 var originalCalculatorValue = RegressionSolutionVariableImpactsCalculator.CalculateVariableImpact(targetValuesPartition, estimatedValuesPartition, out error); 171 if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation."); 169 var originalCalculatorValue = RegressionSolutionVariableImpactsCalculator.CalculateQuality(targetValuesPartition, estimatedValuesPartition); 172 170 173 171 foreach (var variableName in originalVariableOrdering) { … … 179 177 //If the variable isn't used for prediction, it has zero impact. 180 178 if (model.VariablesUsedForPrediction.Contains(variableName)) { 181 impact = RegressionSolutionVariableImpactsCalculator.CalculateImpact(variableName, model, modifiableDataset, rows, targetValuesPartition, originalCalculatorValue, replMethod, factorReplMethod);179 impact = RegressionSolutionVariableImpactsCalculator.CalculateImpact(variableName, model, problemData, modifiableDataset, rows, replMethod, factorReplMethod, targetValuesPartition, originalCalculatorValue); 182 180 } 183 181 impacts.Add(new Tuple<string, double>(variableName, impact));
Note: See TracChangeset
for help on using the changeset viewer.