source: branches/HeuristicLab.Problems.BioBoost/HeuristicLab.Problems.BioBoost/3.3/ProblemDescription/BioBoostProblemData.cs @ 13069

Last change on this file since 13069 was 13069, checked in by gkronber, 7 years ago

#2499: imported source code for HeuristicLab.BioBoost from private repository with some changes

File size: 31.8 KB
Line 
1using System.Data;
2using System.Diagnostics;
3using System.Text.RegularExpressions;
4using HeuristicLab.BioBoost.Data;
5using HeuristicLab.BioBoost.Evaluators;
6using HeuristicLab.BioBoost.Persistence;
7using HeuristicLab.BioBoost.Utils;
8using HeuristicLab.Common;
9using HeuristicLab.Core;
10using HeuristicLab.Data;
11using HeuristicLab.Parameters;
12using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
13using SharpMap.Data;
14using SharpMap.Data.Providers;
15using System;
16using System.Collections.Generic;
17using System.Globalization;
18using System.IO;
19using System.Linq;
20using GeoAPI.Geometries;
21using HeuristicLab.BioBoost.Representation;
22
23namespace HeuristicLab.BioBoost.ProblemDescription {
24
25  [StorableClass]
26  public class BioBoostProblemData : ParameterizedNamedItem {
27
28    #region Events
29    public event EventHandler GeometryChanged;
30    protected void OnGeometryChanged() {
31      var h = GeometryChanged;
32      if (h != null) h(this, EventArgs.Empty);
33    }
34    #endregion
35
36    public new ParameterCollection Parameters {
37      get { return base.Parameters; }
38    }
39
40    // this prevents geometries from being checked by tree traversal
41    private readonly Closure<GeometryFeatureProvider> geometry = new Closure<GeometryFeatureProvider>(Closure<GeometryFeatureProvider>.Encapsulation.Referenced);
42
43    [Storable]
44    public GeometryFeatureProvider Geometry {
45      get { return geometry.Value; }
46      private set { geometry.Value = value; }
47    }
48
49    #region IStorableContent Members
50    public string Filename { get; set; }
51    #endregion
52
53    #region Parameters
54    public ValueParameter<StringArray> LocationNamesParameter {
55      get { return (ValueParameter<StringArray>)Parameters["LocationNames"]; }
56    }
57    public ValueParameter<NeighborhoodTable> NeighborsParameter {
58      get { return (ValueParameter<NeighborhoodTable>)Parameters["Neighbors"]; }
59    }
60    private ValueParameter<ItemCollection<DataItem>> DataItemsParameter {
61      get { return (ValueParameter<ItemCollection<DataItem>>)Parameters["DataItems"]; }
62    }
63    public ValueParameter<ParameterCollection> FeedstockPotentialsParameter {
64      get { return (ValueParameter<ParameterCollection>)Parameters["FeedstockPotentials"]; }
65    }
66    public ValueParameter<DoubleArray> InvestmentCostFactorsParameter {
67      get { return (ValueParameter<DoubleArray>)Parameters["InvestmentCostFactors"]; }
68    }
69    public ValueParameter<DoubleArray> MaintenanceCostFactorsParameter {
70      get { return (ValueParameter<DoubleArray>)Parameters["MaintenanceCostFactors"]; }
71    }
72    public ValueParameter<DoubleArray> FuelCostFactorsParameter {
73      get { return (ValueParameter<DoubleArray>)Parameters["FuelCostFactors"]; }
74    }
75    public ValueParameter<DoubleArray> LaborCostFactorsParameter {
76      get { return (ValueParameter<DoubleArray>)Parameters["LaborCostFactors"]; }
77    }
78    public ValueParameter<DoubleArray> OtherCostFactorsParameter {
79      get { return (ValueParameter<DoubleArray>)Parameters["OtherCostFactors"]; }
80    }
81    public ValueParameter<CheckedItemList<StringValue>> UtilizationsParameter {
82      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["Utilizations"]; }
83    }
84    public ValueParameter<CheckedItemList<StringValue>> TransportTargetsParameter {
85      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["TransportTargets"]; }
86    }
87    public ValueParameter<CheckedItemList<StringValue>> FinalProductsParameter {
88      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["FinalProducts"]; }
89    }
90    public IValueParameter<DoubleMatrix> SplitRegionDistanceFactorsParameter {
91      get { return (IValueParameter<DoubleMatrix>) Parameters["SplitRegionDistanceFactors"]; }
92    }
93    public IValueParameter<StringMatrix> ProductLinksParameter { get { return (IValueParameter<StringMatrix>) Parameters["ProductLinks"]; } }
94    #endregion
95
96    #region Parameter Values
97    public StringArray LocationNames {
98      get { return LocationNamesParameter.Value; }
99      set { LocationNamesParameter.Value = value; }
100    }
101    public NeighborhoodTable Neighbors {
102      get { return NeighborsParameter.Value; }
103      set { NeighborsParameter.Value = value; }
104    }
105    public ItemCollection<DataItem> DataItems {
106      get { return DataItemsParameter.Value; }
107      private set {
108        DataItemsParameter.Value = value;
109        Products = value.OfType<Product>().ToList();
110        Conversions = value.OfType<Conversion>().ToLookup(c => c.Feedstock, c => c);
111        Logistics = value.OfType<Logistic>().ToLookup(l => l.Product, l => l);
112      }
113    }
114    public ParameterCollection FeedstockPotentials {
115      get { return FeedstockPotentialsParameter.Value; }
116      set { FeedstockPotentialsParameter.Value = value; }
117    }
118    public DoubleArray InvestmentCostFactors {
119      get { return InvestmentCostFactorsParameter.Value; }
120      set { InvestmentCostFactorsParameter.Value = value; }
121    }
122    public DoubleArray MaintenanceCostFactors {
123      get { return MaintenanceCostFactorsParameter.Value; }
124      set { MaintenanceCostFactorsParameter.Value = value; }
125    }
126    public DoubleArray FuelCostFactors {
127      get { return FuelCostFactorsParameter.Value; }
128      set { FuelCostFactorsParameter.Value = value; }
129    }
130    public DoubleArray LaborCostFactors {
131      get { return LaborCostFactorsParameter.Value; }
132      set { LaborCostFactorsParameter.Value = value; }
133    }
134    public DoubleArray OtherCostFactors {
135      get { return OtherCostFactorsParameter.Value; }
136      set { OtherCostFactorsParameter.Value = value; }
137    }
138    public CheckedItemList<StringValue> Utilizations {
139      get { return UtilizationsParameter.Value; }
140      set { UtilizationsParameter.Value = value; }
141    }
142    public CheckedItemList<StringValue> TransportTargets {
143      get { return TransportTargetsParameter.Value; }
144      set { TransportTargetsParameter.Value = value; }
145    }
146    public CheckedItemList<StringValue> FinalProducts {
147      get { return FinalProductsParameter.Value; }
148      set { FinalProductsParameter.Value = value; }
149    }
150    public DoubleMatrix SplitRegionDistanceFactors {
151      get { return SplitRegionDistanceFactorsParameter.Value; }
152      set { SplitRegionDistanceFactorsParameter.Value = value; }
153    }
154    public IEnumerable<Product> Products { get; private set; }
155    public ILookup<string, Conversion> Conversions { get; private set; }
156    public ILookup<string, Logistic> Logistics { get; private set; }
157    public StringMatrix ProductLinks {
158      get { return ProductLinksParameter.Value; }
159      set { ProductLinksParameter.Value = value; }
160    }
161    #endregion
162
163    #region Construction & Cloning
164
165    [StorableConstructor]
166    protected BioBoostProblemData(bool isDeserializing) : base(isDeserializing) { }
167    protected BioBoostProblemData(BioBoostProblemData orig, Cloner cloner)
168      : base(orig, cloner) {
169      Geometry = orig.Geometry;
170      DataItems = DataItems; // invoke setter to populate Products, Conversions and Logistics
171    }
172    public BioBoostProblemData() {
173      Parameters.Add(new ValueParameter<StringArray>("LocationNames", "The names of the individual locations."));
174      Parameters.Add(new ValueParameter<NeighborhoodTable>("Neighbors", "The table of neighborhood degrees."));
175      Parameters.Add(new ValueParameter<ItemCollection<DataItem>>("DataItems", "Data descriptors, e.g. for products or logistics and conversion processes."));
176      Parameters.Add(new ValueParameter<ParameterCollection>("FeedstockPotentials", "Imported feedstock potentials."));
177      Parameters.Add(new ValueParameter<DoubleArray>("InvestmentCostFactors", "The scaling factors for investment costs."));
178      Parameters.Add(new ValueParameter<DoubleArray>("MaintenanceCostFactors", "The scaling factors for maintenance costs."));
179      Parameters.Add(new ValueParameter<DoubleArray>("FuelCostFactors", "The scaling factors for fuel costs."));
180      Parameters.Add(new ValueParameter<DoubleArray>("LaborCostFactors", "The scaling factors for labor costs."));
181      Parameters.Add(new ValueParameter<DoubleArray>("OtherCostFactors", "The scaling factors for other costs."));
182      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("Utilizations", "The possible utilization vectors."));
183      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("TransportTargets", "The possible transport target vectors."));
184      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("FinalProducts", "The list of possible final products used for quality evaluation."));
185      Parameters.Add(new ValueParameter<DoubleMatrix>("SplitRegionDistanceFactors", "Describes the reduction in distance if the region would be split into different numbers of smaller regions"));
186      Parameters.Add(new ValueParameter<StringMatrix>("ProductLinks", "List of tuples of product names that are linked in the logistic chain"));
187    }
188    public override IDeepCloneable Clone(Cloner cloner) {
189      return new BioBoostProblemData(this, cloner);
190    }
191    [StorableHook(HookType.AfterDeserialization)]
192    public void AfterDeserialization() {
193      DataItems = DataItems; // invoke setter to populate Products, Conversions and Logistics
194      TryUpgradeStringArrayToCheckedItemList("Utilizations");
195      TryUpgradeStringArrayToCheckedItemList("TransportTargets");
196      if (!Parameters.ContainsKey("SplitRegionDistanceFactors"))
197        Parameters.Add(new ValueParameter<DoubleMatrix>("SplitRegionDistanceFactors", "Describes the reduction in distance if the region would be split into different numbers of smaller regions"));
198      if (!Parameters.ContainsKey("ProductLinks"))
199        Parameters.Add(new ValueParameter<StringMatrix>("ProductLinks", "List of tuples of product names that are linked in the logistic chain"));
200      if (!Parameters.ContainsKey("FinalProducts"))
201        Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("FinalProducts", "The list of possible final products used for quality evaluation."));
202    }
203
204    private void TryUpgradeStringArrayToCheckedItemList(string parameterName) {
205      var oldParameter = Parameters[parameterName] as ValueParameter<StringArray>;
206      if (oldParameter != null) {
207        var newParameter = new ValueParameter<CheckedItemList<StringValue>>(oldParameter.Name, oldParameter.Description);
208        Parameters.Remove(parameterName);
209        foreach (var s in oldParameter.Value)
210          newParameter.Value.Add(new StringValue(s), true);
211        Parameters.Add(newParameter);
212      }
213    }
214    #endregion
215
216    public override void CollectParameterValues(IDictionary<string, IItem> values) {
217      base.CollectParameterValues(values);
218      foreach (var di in DataItems) {
219        di.CollectParameterValues(values);
220      }
221    }
222
223    #region Data Parsing & Import
224    private void AddParameter<TValue>(string name, TValue value) where TValue : class, IItem {
225      Parameters.Remove(name);
226      Parameters.Add(new ValueParameter<TValue>(name, value));
227    }
228
229    public void LoadFeedstockPotentials(string filename) {
230      var feedstockPotentials = new List<Tuple<string, List<double>>>();
231      var regionNames = new List<string>();
232      bool headerDone = false;
233      foreach (var line in File.ReadLines(filename)) {
234        var fields = line.Split(';');
235        if (!headerDone) {
236          feedstockPotentials.AddRange(
237            fields.Skip(1).Select(feedstockName => Tuple.Create(feedstockName, new List<double>())));
238          headerDone = true;
239        } else {
240          regionNames.Add(fields[0]);
241          for (int i = 1; i < fields.Length; i++) {
242            double value;
243            double.TryParse(fields[i], NumberStyles.Float, NumberFormatInfo.InvariantInfo, out value);
244            feedstockPotentials[i - 1].Item2.Add(value);
245          }
246        }
247      }
248
249      if (LocationNames.Length > 0) {
250        var newRegionNames = new HashSet<string>(regionNames);
251        newRegionNames.ExceptWith(LocationNames);
252        var indexTable = LocationNames.Concat(newRegionNames)
253          .Select((n, i) => new { idx = i, name = n })
254          .ToDictionary(p => p.name, p => p.idx);
255        if (newRegionNames.Count > 0)
256          LocationNames = new StringArray(LocationNames.Concat(newRegionNames).ToArray());
257        foreach (var fp in feedstockPotentials) {
258          var potentials = new DoubleArray(indexTable.Count);
259          var potentialName = LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(fp.Item1);
260          for (int i = 0; i < regionNames.Count; i++) {
261            potentials[indexTable[regionNames[i]]] = fp.Item2[i];
262          }
263          FeedstockPotentials.Remove(potentialName);
264          FeedstockPotentials.Add(new ValueParameter<DoubleArray>(potentialName, potentials));
265        }
266      } else {
267        LocationNames = new StringArray(regionNames.ToArray());
268        foreach (var fp in feedstockPotentials) {
269          FeedstockPotentials.Add(new ValueParameter<DoubleArray>(LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(fp.Item1), new DoubleArray(fp.Item2.ToArray())));
270        }
271      }
272
273
274      Configure();
275    }
276
277    public void LoadShapeFile(string filename) {
278      Geometry = ShapeFileLoader.LoadShapeFile(filename);
279      OnGeometryChanged();
280    }
281
282    public void AddDoubleArrayParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
283      int i = 0;
284      foreach (var f in fields) {
285        if (i >= names.Count) break;
286        var subfields = f.Split('/');
287        var values = new List<double>();
288        foreach (var sf in subfields) {
289          double d;
290          Double.TryParse(sf, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
291          values.Add(d);
292        }
293        AddParameter(names[i] + prefix, new DoubleArray(values.ToArray()));
294        i++;
295      }
296    }
297
298    public void AddDoubleValueParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
299      int i = 0;
300      foreach (var f in fields) {
301        if (i >= names.Count) break;
302        double d;
303        Double.TryParse(f, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
304        AddParameter(names[i] + prefix, new DoubleValue(d));
305        i++;
306      }
307    }
308
309    public void AddIntValueParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
310      int i = 0;
311      foreach (var f in fields) {
312        if (i >= names.Count) break;
313        int n;
314        Int32.TryParse(f, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out n);
315        AddParameter(names[i] + prefix, new IntValue(n));
316        i++;
317      }
318    }
319
320    public void LoadDistanceMatrix(string filename) {
321      var newLocationNames = new HashSet<string>();
322      var tuples = new List<Tuple<string, string, double>>();
323      var matrixName = "";
324      foreach (var line in File.ReadLines(filename)) {
325        var l = line.Trim();
326        if (l.StartsWith("#")) {
327          matrixName = l.TrimStart("# ".ToCharArray());
328        } else {
329          var fields = l.Split(';').ToList();
330          newLocationNames.Add(fields[0]);
331          newLocationNames.Add(fields[1]);
332          double d;
333          Double.TryParse(fields[2], NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
334          tuples.Add(Tuple.Create(fields[0], fields[1], d));
335        }
336      }
337      newLocationNames.ExceptWith(LocationNames);
338      if (newLocationNames.Count > 0) {
339        LocationNames = new StringArray(LocationNames.Concat(newLocationNames).ToArray());
340        foreach (ValueParameter<DoubleArray> p in FeedstockPotentials) {
341          p.Value = new DoubleArray(p.Value.Concat(new double[newLocationNames.Count]).ToArray());
342        }
343      }
344      var dm = new DistanceMatrix();
345      dm.Load(tuples, LocationNames, -1);
346      matrixName += "DistanceMatrix";
347      if (Parameters.ContainsKey(matrixName))
348        Parameters.Remove(matrixName);
349      Parameters.Add(new ValueParameter<DistanceMatrix>(matrixName, dm));
350      Configure();
351    }
352
353    public void LoadSplitRegionDistanceFactors(string filename) {
354      // format: regionName;factor2;factor3;factor4;...
355      var newLocationNames = new HashSet<string>();
356      var splits = new Dictionary<string, List<double>>();
357      int maxN = 0;
358      foreach (var line in File.ReadLines(filename)) {
359        var l = line.Trim();
360        if (l.ToUpper().StartsWith("NUTS")) continue;
361        var fields = line.Split(';').ToList();
362        newLocationNames.Add(fields[0]);
363        splits.Add(fields[0], fields.Skip(1).Select(s => ParseDouble(s, 1)).ToList());
364        maxN = Math.Max(fields.Count, maxN);
365      }
366      newLocationNames.ExceptWith(LocationNames);
367      if (newLocationNames.Count > 0) {
368        LocationNames = new StringArray(LocationNames.Concat(newLocationNames).ToArray());
369        SplitRegionDistanceFactors = new DoubleMatrix(LocationNames.Length, maxN - 1,
370          Enumerable.Range(2, maxN - 1).Select(d => d.ToString(NumberFormatInfo.InvariantInfo)),
371          LocationNames);
372        for (int row = 0; row < SplitRegionDistanceFactors.Rows; row++) {
373          List<double> values;
374          if (splits.TryGetValue(LocationNames[row], out values)) {
375            for (int col = 0; col < values.Count; col++) {
376              SplitRegionDistanceFactors[row, col] = values[col];
377            }
378          }
379        }
380      }
381    }
382
383    public static double ParseDouble(string s, double defaultValue = 0) {
384      double d;
385      if (Double.TryParse(s, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d))
386        return d;
387      return defaultValue;
388    }
389
390    public void RemoveRegions(Regex regex) {
391      RemoveRegions(LocationNames.Where(n => regex.IsMatch(n)));
392    }
393
394    public void KeepRegions(Regex regex) {
395      RemoveRegions(LocationNames.Where(n => !regex.IsMatch(n)));
396    }
397
398    public void RemoveRegions(IEnumerable<string> regionNames) {
399      var regionNameSet = new HashSet<string>(regionNames);
400      var removedIndices =
401        LocationNames.Select((r, i) => new { r, i })
402                     .Where(ri => regionNameSet.Contains(ri.r))
403                     .Select(ri => ri.i)
404                     .ToList();
405      // remove region names
406      int nLocationNamesBefore = LocationNames.Length;
407      LocationNames = new StringArray(LocationNames.RemoveAtIndices(removedIndices).ToArray());
408      int nLocationNamesAfter = LocationNames.Length;
409      Debug.Assert(nLocationNamesBefore - removedIndices.Count == nLocationNamesAfter);
410      // remove region potentials
411      foreach (var potentialName in FeedstockPotentials.Select(p => p.Name).ToList()) {
412        var potentials = FeedstockPotentials[potentialName];
413        FeedstockPotentials.Remove(potentialName);
414        FeedstockPotentials.Add(
415          new ValueParameter<DoubleArray>(
416            potentialName,
417            new DoubleArray(((ValueParameter<DoubleArray>)potentials).Value.RemoveAtIndices(removedIndices).ToArray())));
418      }
419      // remove region distances
420      foreach (var distanceMatrixParam in Parameters.Where(p => p.Name.EndsWith("DistanceMatrix"))) {
421        var valueParam = distanceMatrixParam as IValueParameter;
422        if (valueParam != null && typeof(DistanceMatrix).IsAssignableFrom(valueParam.DataType)) {
423          var matrix = (DistanceMatrix)valueParam.Value;
424          valueParam.Value = matrix.CopyAndRemoveIndices(removedIndices);
425        }
426      }
427      // remove region neighbors
428      if (Neighbors != null)
429        Neighbors = Neighbors.CopyAndRemoveIndices(removedIndices);
430      // remove shapes & and create new geometry (copy on write)
431      var locationNameset = new HashSet<string>(LocationNames);
432      var keptRows = Geometry.Features.Cast<FeatureDataRow>()
433        .Where(row => locationNameset.Contains(row["NUTS_ID"]))
434        .ToList();
435      var newFeatures = new FeatureDataTable();
436      foreach (DataColumn column in Geometry.Features.Columns) {
437        newFeatures.Columns.Add(column.ColumnName, column.DataType);
438      }
439      foreach (FeatureDataRow row in keptRows) {
440        var newRow = (FeatureDataRow)newFeatures.LoadDataRow(row.ItemArray, LoadOption.OverwriteChanges);
441        newRow.Geometry = (IGeometry) row.Geometry.Clone();
442      }
443      newFeatures.AcceptChanges();
444      Geometry = new GeometryFeatureProvider(newFeatures);
445      OnGeometryChanged();
446      Configure();
447    }
448
449    public void LoadNeighbors(string filename) {
450      var locationNames = LocationNames.Select((n, i) => new { n, i }).ToDictionary(p => p.n, p => p.i);
451      var neighbors = new NeighborhoodTable();
452      foreach (var line in File.ReadLines(filename)) {
453        var fields = line.Split(';');
454        int id1, id2, degree;
455        if (locationNames.TryGetValue(fields[0], out id1) &&
456            locationNames.TryGetValue(fields[1], out id2) &&
457            Int32.TryParse(fields[2], out degree))
458          neighbors.AddEntry(id1, id2, degree);
459      }
460      neighbors.Check();
461      Neighbors = neighbors;
462    }
463
464    public void LoadDescriptors(string filename) {
465      var newItems = new ItemCollection<DataItem>(DataItem.ParseFile(filename));
466      if (DataItems != null) {
467        foreach (var oldItem in DataItems) {
468          if (!newItems.Any(i => i.IsEquivalentTo(oldItem)))
469            newItems.Add(oldItem);
470        }
471      }
472      DataItems = newItems;
473      Configure();
474    }
475
476    private void Configure() {
477      DistributeCostFactors();
478      DiscoverSolutionSpace();
479    }
480
481    private void DistributeCostFactors() {
482      int length = LocationNames.Length;
483      InvestmentCostFactors = new DoubleArray(length);
484      MaintenanceCostFactors = new DoubleArray(length);
485      FuelCostFactors = new DoubleArray(length);
486      LaborCostFactors = new DoubleArray(length);
487      OtherCostFactors = new DoubleArray(length);
488      var costFactorDescriptors = DataItems.OfType<CostFactors>().FirstOrDefault();
489      if (costFactorDescriptors == null) return;
490      for (int i = 0; i < length; i++) {
491        var countryCode = LocationNames[i].Substring(0, 2);
492        InvestmentCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Investment, countryCode, 1);
493        MaintenanceCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Maintenance, countryCode, 1);
494        FuelCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Fuel, countryCode, 1);
495        LaborCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Labor, countryCode, 1);
496        OtherCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Other, countryCode, 1);
497      }
498    }
499
500    private void DiscoverSolutionSpace() {
501      if (DataItems == null || DataItems.Count == 0) return;
502      var products = Products;
503      var logistics = Logistics;
504      var conversions = DataItems.OfType<Conversion>().ToList();
505
506      // perform conceptual logistic simulation
507      var usableProducts = products
508        .Where(product => FeedstockPotentials.ContainsKey(LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(product.Label)))
509        .ToList();
510      var transportableProducts = usableProducts
511        .Where(p => logistics.Contains(p.Label))
512        .ToList();
513      var feasibleDecentralConversions = conversions
514        .Where(c => transportableProducts.Any(p => p.Label == c.Feedstock))
515        .ToList();
516      var possibleIntermediates = feasibleDecentralConversions
517        .Aggregate(Enumerable.Empty<string>(),
518                   (intermediates, conv) => intermediates.Concat(conv.Products.Select(p => p.Name)))
519        .ToList();
520      var transportableIntermediates = possibleIntermediates
521        .Where(logistics.Contains) // && intermediate exists as product?
522        .ToList();
523      var feasibleCentralConversions = conversions
524        .Where(c => transportableIntermediates.Any(i => i == c.Feedstock))
525        .ToList();
526      var finalProductLists = feasibleCentralConversions
527        .Select(c => c.Products.Where(p => { var v = (p.Value as DoubleValue); return v != null && v.Value > 0; }).Select(p => p.Name));
528      var finalProductList = finalProductLists.Aggregate(Enumerable.Empty<string>(), (x, y) => x.Concat(y));
529      var uniqueFinalProducts = finalProductList.GroupBy(p => p).Select(p => p.Key);
530      var valuableFinalProducts = uniqueFinalProducts.Where(productName => Products.Any(p => p.Label == productName && p.Price > 0));
531      var finalProducts = valuableFinalProducts.ToList();
532
533      // TODO: forward simulation sanity checks
534      // non-sold products
535      // non-(transported/converted/sold) intermediates
536      // non-transported/converted feedstocks
537
538      // reverse pruning
539      // select only central conversions for which valuable end-products exist
540      var necessaryCentralConversions = feasibleCentralConversions
541        .Where(c => c.Products.Any(p => products.Any(p1 => p1.Label == p.Name && p1.Price > 0)))
542        .ToList();
543      // transport only intermediates that need to be converted
544      var neccessaryIntermediates = transportableIntermediates
545        .Where(i => necessaryCentralConversions.Any(c => c.Feedstock == i))
546        .ToList();
547      // select only intermediates that either
548      // - need to transported and are further processed or
549      // - can be sold
550      var feasibleIntermediates = neccessaryIntermediates.Concat(
551        possibleIntermediates.Where(i => products.Any(p => p.Label == i && p.Price > 0)))
552        .ToList();
553      // produce only selected intermediates
554      var necessaryDecentralConversions = feasibleDecentralConversions
555        .Where(c => c.Products.Any(p => feasibleIntermediates.Contains(p.Name)))
556        .ToList();
557      // transport and buy only feedstocks that are converted and sold
558      var necessaryFeedstocks = necessaryDecentralConversions
559        .Select(c => c.Feedstock).ToList();
560
561      // enable feedstocks directly to central conversion? Yes, it automatically becomes a decentral conversion
562      // TODO: allow more than two echelons?
563
564      // configure search space:
565      UpdateSearchSpace(Utilizations, necessaryFeedstocks);
566      UpdateSearchSpace(TransportTargets, necessaryFeedstocks.Concat(neccessaryIntermediates));
567      UpdateSearchSpace(FinalProducts, finalProducts);
568
569      var q =
570        (from p1 in necessaryFeedstocks.Concat(neccessaryIntermediates)
571          from p2 in neccessaryIntermediates
572          where conversions.Any(c => c.Feedstock == p1 && c.Products.ContainsKey(p2))
573          select new {p1, p2}).Concat(
574        from p1 in neccessaryIntermediates
575        from p2 in finalProducts
576        where conversions.Any(c => c.Feedstock == p1 && c.Products.ContainsKey(p2))
577        select new {p1, p2})
578       .Distinct().ToList();
579      var productLinks = new StringMatrix(q.Count, 2);
580      for (int i = 0; i < q.Count; i++) {
581        productLinks[i, 0] = q[i].p1;
582        productLinks[i, 1] = q[i].p2;
583      }
584      ProductLinks = productLinks;
585
586    }
587
588    private void UpdateSearchSpace(CheckedItemList<StringValue> choices, IEnumerable<string> requirements) {
589      var req = new HashSet<string>(requirements);
590      for (int i = 0; i < choices.Count; i++) {
591        if (req.Contains(choices[i].Value)) {
592          req.Remove(choices[i].Value); // already there, not required anymore
593        } else {
594          choices.SetItemCheckedState(i, false);
595        }
596      }
597      foreach (var r in req) {
598        choices.Add(new StringValue(r), true);
599      }
600    }
601
602    private static double GetDoubleValueValue(ValueParameterCollection parameters, string name, double defaultValue) {
603      IValueParameter param;
604      if (!parameters.TryGetValue(name, out param)) return defaultValue;
605      var doubleValue = param.Value as DoubleValue;
606      if (doubleValue == null) return defaultValue;
607      return doubleValue.Value;
608    }
609
610    #endregion
611
612    #region Data Consistency Checks
613    public string CheckGeoData() {
614      var warnings = new List<string>();
615      CheckDistanceMatrices(warnings);
616      var shapeNames = CheckShapeNames(warnings);
617      var locationNames = CheckLocationNames(warnings);
618      FindMissing(warnings, locationNames, shapeNames, "locations without shapes");
619      FindMissing(warnings, shapeNames, locationNames, "shapes without data");
620      var totalPotentials = new double[LocationNames.Length];
621      foreach (var p in FeedstockPotentials.OfType<ValueParameter<DoubleArray>>()) {
622        totalPotentials = totalPotentials.Zip(p.Value, (a, b) => a + b).ToArray();
623        FindZeros(warnings, p.Value, p.Name);
624      }
625      FindZeros(warnings, totalPotentials, "any potential");
626      return string.Join("\n\n", warnings);
627    }
628
629    private void FindZeros(List<string> warnings, IEnumerable<double> values, string name) {
630      var zeros = values.Select((v, i) => new { v, i }).Where(p => p.v == 0).Select(p => LocationNames[p.i]).ToList();
631      if (zeros.Count > 0)
632        warnings.Add(string.Format("{0} regions with zero of {1}:\n  {2}",
633          zeros.Count, name, string.Join(", ", zeros)));
634    }
635
636    private static void FindMissing(List<string> warnings, IEnumerable<string> items, IEnumerable<string> other, string message) {
637      var missingItems = new HashSet<string>(items);
638      missingItems.ExceptWith(other);
639      if (missingItems.Count > 0)
640        warnings.Add(string.Format("{0} {1}:\n  {2}",
641          missingItems.Count, message, string.Join(", ", missingItems)));
642    }
643
644    private HashSet<string> CheckLocationNames(List<string> warnings) {
645      var duplicates = new List<String>();
646      var locationNames = new HashSet<string>();
647      foreach (var n in LocationNames) {
648        if (locationNames.Contains(n))
649          duplicates.Add(n);
650        locationNames.Add(n);
651      }
652      if (duplicates.Count > 0)
653        warnings.Add(string.Format("{0} duplicate location names:\n  {1}",
654          duplicates.Count, string.Join(", ", duplicates)));
655      return locationNames;
656    }
657
658    private HashSet<string> CheckShapeNames(List<string> warnings) {
659      var duplicates = new List<string>();
660      var shapeNames = new HashSet<string>();
661      foreach (FeatureDataRow row in Geometry.Features.Rows) {
662        var n = (string)row["NUTS_ID"];
663        if (shapeNames.Contains(n))
664          duplicates.Add(n);
665        shapeNames.Add(n);
666      }
667      if (duplicates.Count > 0)
668        warnings.Add(string.Format("{0} duplicate shape names:\n  {1}",
669          duplicates.Count, String.Join(", ", duplicates)));
670      return shapeNames;
671    }
672
673    private void CheckDistanceMatrices(List<string> warnings) {
674      foreach (var distanceMatrixParam in Parameters.Where(p => p.Name.EndsWith("DistanceMatrix"))) {
675        var valueParam = distanceMatrixParam as IValueParameter;
676        if (valueParam != null && typeof(DistanceMatrix).IsAssignableFrom(valueParam.DataType))
677          FindUnconnectedRegions((DistanceMatrix)valueParam.Value, warnings);
678      }
679    }
680
681    private void FindUnconnectedRegions(DistanceMatrix dm, List<string> warnings) {
682      var unconnected = new List<string>();
683      var hardlyConnected = new List<string>();
684      for (int i = 0; i < dm.Size; i++) {
685        var nConnected = 0;
686        for (int j = 0; j < i; j++) {
687          if (dm[i, j] > 0) nConnected++;
688        }
689        if (nConnected == 0)
690          unconnected.Add(LocationNames[i]);
691        else if (1.0 * nConnected < 1.0 * LocationNames.Length / 100)
692          hardlyConnected.Add(LocationNames[i]);
693      }
694      if (unconnected.Count > 0)
695        warnings.Add(string.Format("{0} unconnected regions:\n  {1}",
696          unconnected.Count, string.Join(", ", unconnected)));
697      if (hardlyConnected.Count > 0)
698        warnings.Add(string.Format("{0} connected to fewer than 1% of regions:\n  {1}",
699          hardlyConnected.Count, string.Join(", ", hardlyConnected)));
700    }
701
702    // TODO: check consistency of conversion process
703    #endregion
704  }
705}
706
Note: See TracBrowser for help on using the repository browser.