source: branches/3026_IntegrationIntoSymSpace/HeuristicLab.JsonInterface/Converters/RegressionProblemDataConverter.cs @ 18033

Last change on this file since 18033 was 18033, checked in by dpiringe, 11 months ago

#3026

  • fixed the injection of VariableRanges in RegressionProblemDataConverter (now: the entries in IntervalCollection get replaced instead of the whole collection object)
File size: 11.7 KB
Line 
1using System;
2using System.Collections;
3using System.Collections.Generic;
4using System.Linq;
5using System.Reflection;
6using System.Text;
7using System.Threading.Tasks;
8using HeuristicLab.Core;
9using HeuristicLab.Data;
10using HeuristicLab.Parameters;
11using HeuristicLab.Problems.DataAnalysis;
12using Newtonsoft.Json.Linq;
13
14namespace HeuristicLab.JsonInterface {
15  public class RegressionProblemDataConverter : BaseConverter {
16    #region Constants
17    private const BindingFlags flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance;
18    private const string TestPartition = "TestPartition";
19    private const string TrainingPartition = "TrainingPartition";
20    private const string TargetVariable = "TargetVariable";
21    private const string AllowedInputVariables = "AllowedInputVariables";
22    private const string Dataset = "Dataset";
23    private const string VariableValues = "variableValues";
24    private const string VariableNames = "variableNames";
25    private const string InputVariables = "InputVariables";
26    private const string VariableRanges = "VariableRanges";
27    private const string Rows = "rows";
28    private const string Value = "value";
29    private const string Parameters = "parameters";
30    private const string CheckedItemList = "CheckedItemList";
31    #endregion
32
33    public override int Priority => 20;
34
35    public override bool CanConvertType(Type t) =>
36      HEAL.Attic.Mapper.StaticCache.GetType(new Guid("EE612297-B1AF-42D2-BF21-AF9A2D42791C")).IsAssignableFrom(t);
37
38    public override void Inject(IItem item, IJsonItem data, IJsonItemConverter root) {
39
40      dynamic regressionProblemData = (dynamic)item;
41
42      DoubleMatrixJsonItem dataset = null;
43      StringJsonItem targetVariable = null;
44      IntRangeJsonItem testPartition = null;
45      IntRangeJsonItem trainingPartition = null;
46      StringArrayJsonItem allowedInputVariables = null;
47
48
49      // search first for the items (cache them, because the
50      // order is important for injection)
51      foreach (var child in data.Children) {
52
53        if (child.Path.EndsWith(Dataset))
54          dataset = child as DoubleMatrixJsonItem;
55        else if (child.Path.EndsWith(TargetVariable))
56          targetVariable = child as StringJsonItem;
57        else if (child.Path.EndsWith(TestPartition))
58          testPartition = child as IntRangeJsonItem;
59        else if (child.Path.EndsWith(TrainingPartition))
60          trainingPartition = child as IntRangeJsonItem;
61        else if (child.Path.EndsWith(AllowedInputVariables))
62          allowedInputVariables = child as StringArrayJsonItem;
63
64      }
65
66      // check data
67      if(!dataset.ColumnNames.Any(x => x == targetVariable.Value)) {
68        throw new Exception($"The value of the target variable ('{targetVariable.Value}') has no matching row name value of the dataset.");
69      }
70
71      foreach(var v in allowedInputVariables.Value) {
72        if(!dataset.ColumnNames.Any(x => x == v))
73          throw new Exception($"The value of the input variable ('{v}') has no matching row name value of the dataset.");
74      }
75
76      // inject the value of the items
77      SetDataset(regressionProblemData, dataset);
78      SetTargetVariable(regressionProblemData, targetVariable);
79      SetAllowedInputVariables(regressionProblemData, allowedInputVariables, dataset);
80      SetTestPartition(regressionProblemData, testPartition);
81      SetTrainingPartition(regressionProblemData, trainingPartition);
82      SetVariableRanges(regressionProblemData, dataset);
83    }
84
85    public override IJsonItem Extract(IItem value, IJsonItemConverter root) {
86      IJsonItem item = new EmptyJsonItem() {
87        Name = value.ItemName,
88        Description = value.ItemDescription
89      };
90
91      IJsonItem ds = GetDataset(value);
92      if(ds != null)
93        item.AddChildren(ds);
94
95      item.AddChildren(GetTestPartition(value));
96      item.AddChildren(GetTrainingPartition(value));
97      item.AddChildren(GetTargetVariable(value));
98      item.AddChildren(GetAllowedInputVariables(value));
99      return item;
100    }
101
102    #region Inject Helper
103    private void SetDataset(dynamic regressionProblemData, DoubleMatrixJsonItem item) {
104      if (item != null) {
105        var dictTmp = new Dictionary<string, IList>();
106        int c = 0;
107        foreach (var col in item.ColumnNames) {
108          dictTmp.Add(col, new List<double>(item.Value[c]));
109          ++c;
110        }
111
112        object dataset = (object)regressionProblemData.Dataset;
113        var rows = dataset.GetType().GetField(Rows, flags);
114        rows.SetValue(dataset, item.Value[0].Length);
115
116        var variableNames = dataset.GetType().GetField(VariableNames, flags);
117        variableNames.SetValue(dataset, item.ColumnNames);
118
119        var dataInfo = dataset.GetType().GetField(VariableValues, flags);
120        dataInfo.SetValue(dataset, dictTmp);
121      }
122    }
123
124    private void SetTestPartition(dynamic regressionProblemData, IntRangeJsonItem item) {
125      if (item != null) {
126        regressionProblemData.TestPartition.Start = item.MinValue;
127        regressionProblemData.TestPartition.End = item.MaxValue;
128      }
129    }
130
131    private void SetTrainingPartition(dynamic regressionProblemData, IntRangeJsonItem item) {
132      if (item != null) {
133        regressionProblemData.TrainingPartition.Start = item.MinValue;
134        regressionProblemData.TrainingPartition.End = item.MaxValue;
135      }
136    }
137
138    private void SetTargetVariable(dynamic regressionProblemData, StringJsonItem item) {
139      if (item != null) {
140        var param = (IConstrainedValueParameter<StringValue>)regressionProblemData.TargetVariableParameter;
141        StringValue v = param.Value;
142        FieldInfo fi = v.GetType().GetField(Value, flags);
143        fi.SetValue(v, item.Value);
144      }
145    }
146
147    private void SetAllowedInputVariables(dynamic regressionProblemData, StringArrayJsonItem item, IMatrixJsonItem matrix) {
148      if (item != null && regressionProblemData is IParameterizedItem p) {
149        var regProbDataType = ((ParameterizedNamedItem)regressionProblemData).GetType(); //RegressionProblemData
150
151        var parameterizedNamedItemType = regProbDataType.BaseType.BaseType;
152
153        // reset parameter
154        var parametersInfo = parameterizedNamedItemType.GetField(Parameters, flags);
155        ParameterCollection col = (ParameterCollection)parametersInfo.GetValue((object)regressionProblemData);
156        var oldParam = (FixedValueParameter<ReadOnlyCheckedItemList<StringValue>>)col[InputVariables];
157        var value = oldParam.Value;
158        PropertyInfo listInfo = value.GetType().GetProperty(CheckedItemList, flags);
159        CheckedItemList<StringValue> checkedItemList = (CheckedItemList<StringValue>)listInfo.GetValue(value);
160        checkedItemList.Clear();
161
162        // add list items and set their check state (based on allowed input variables)
163        foreach(var i in matrix.ColumnNames) {
164          bool isChecked = false;
165          foreach(var x in item.Value)
166            isChecked = isChecked || (x == i);
167          checkedItemList.Add(new StringValue(i).AsReadOnly(), isChecked);
168        }
169      }
170    }
171
172    private void SetVariableRanges(dynamic regressionProblemData, DoubleMatrixJsonItem item) {
173      if (item != null) {
174        IntervalCollection variableRanges = (IntervalCollection)regressionProblemData.VariableRanges;
175
176        foreach(var kvp in variableRanges.GetDictionary()) {
177          variableRanges.DeleteInterval(kvp.Key);
178        }
179       
180        int count = 0;
181        foreach (var column in item.ColumnNames) {
182          var doubleValuesForColumn = item.Value[count];
183          if (doubleValuesForColumn.Any(x => double.IsNaN(x))) // add a NaN interval if any NaN value exists
184            variableRanges.AddInterval(column, new Interval(double.NaN, double.NaN));
185          else
186            variableRanges.AddInterval(column, new Interval(doubleValuesForColumn.Min(), doubleValuesForColumn.Max()));
187          count++;
188        }
189      }
190    }
191
192    private void SetShapeConstraints() {
193      // TODO
194    }
195    #endregion
196
197    #region Extract Helper
198    private IJsonItem GetDataset(IItem item) {
199      dynamic val = (dynamic)item;
200      object dataset = (object)val.Dataset;
201      FieldInfo dataInfo = dataset.GetType().GetField(VariableValues, flags);
202
203      if (dataInfo.GetValue(dataset) is Dictionary<string, IList> dict) {
204        IEnumerator it = dict.Values.First()?.GetEnumerator();
205
206        if(it != null) {
207          if(it.MoveNext() && it.Current is double) {
208            CreateMatrix(dict, out IList<string> columnNames, out double[][] mat);
209            return new DoubleMatrixJsonItem() {
210              Name = Dataset,
211              Value = mat,
212              ColumnNames = columnNames,
213              Minimum = double.MinValue,
214              Maximum = double.MaxValue
215            };
216          } else if(it.Current is int) {
217            CreateMatrix(dict, out IList<string> columnNames, out int[][] mat);
218            return new IntMatrixJsonItem() {
219              Name = Dataset,
220              Value = mat,
221              ColumnNames = columnNames,
222              Minimum = int.MinValue,
223              Maximum = int.MaxValue
224            };
225          } else if (it.Current is bool) {
226            CreateMatrix(dict, out IList<string> columnNames, out bool[][] mat);
227            return new BoolMatrixJsonItem() {
228              Name = Dataset,
229              Value = mat,
230              ColumnNames = columnNames
231            };
232          }
233        }
234      }
235      return null;
236    }
237   
238    private void CreateMatrix<T>(Dictionary<string, IList> dict, out IList<string> columnNames, out T[][] matrix) {
239      int cols = dict.Count, rows = 0, c = 0;
240      columnNames = new List<string>();
241      matrix = new T[cols][];
242      foreach (var x in dict) {
243        rows = Math.Max(rows, x.Value.Count);
244        columnNames.Add(x.Key);
245
246        matrix[c] = new T[rows];
247
248        int r = 0;
249        foreach (var callValue in x.Value) {
250          matrix[c][r] = (T)callValue;
251          ++r;
252        }
253        ++c;
254      }
255    }
256
257    private IJsonItem GetTestPartition(IItem item) {
258      dynamic val = (dynamic)item;
259      var trainingPartition = (IntRange)val.TrainingPartition;
260      var testPartition = (IntRange)val.TestPartition;
261      return new IntRangeJsonItem() {
262        Name = TestPartition,
263        MinValue = testPartition.Start,
264        MaxValue = testPartition.End,
265        Minimum = 0,
266        Maximum = int.MaxValue //Math.Max(testPartition.End, trainingPartition.End)
267      };
268    }
269
270    private IJsonItem GetTrainingPartition(IItem item) {
271      dynamic val = (dynamic)item;
272      var trainingPartition = (IntRange)val.TrainingPartition;
273      var testPartition = (IntRange)val.TestPartition;
274      return new IntRangeJsonItem() {
275        Name = TrainingPartition,
276        MinValue = trainingPartition.Start,
277        MaxValue = trainingPartition.End,
278        Minimum = 0,
279        Maximum = int.MaxValue //Math.Max(testPartition.End, trainingPartition.End)
280      };
281    }
282
283
284    private IJsonItem GetTargetVariable(IItem item) {
285      var vars = (IEnumerable<StringValue>)((dynamic)item).InputVariables;
286      return new StringJsonItem() {
287        Name = TargetVariable,
288        Value = (string)((dynamic)item).TargetVariable,
289        ConcreteRestrictedItems = vars.Select(x => x.Value)
290      };
291    }
292
293    private IJsonItem GetAllowedInputVariables(IItem item) {
294      var vars = (IEnumerable<StringValue>)((dynamic)item).InputVariables;
295      return new StringArrayJsonItem() {
296        Name = AllowedInputVariables,
297        Value = ((IEnumerable<string>)((dynamic)item).AllowedInputVariables).ToArray(),
298        ConcreteRestrictedItems = vars.Select(x => x.Value)
299      };
300    }
301    #endregion
302  }
303}
Note: See TracBrowser for help on using the repository browser.