source: branches/HeuristicLab.Problems.BioBoost/HeuristicLab.Problems.BioBoost/3.3/ProblemDescription/BioBoostProblemData.cs @ 13069

using System.Data;
using System.Diagnostics;
using System.Text.RegularExpressions;
using HeuristicLab.BioBoost.Data;
using HeuristicLab.BioBoost.Evaluators;
using HeuristicLab.BioBoost.Persistence;
using HeuristicLab.BioBoost.Utils;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using SharpMap.Data;
using SharpMap.Data.Providers;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using GeoAPI.Geometries;
using HeuristicLab.BioBoost.Representation;

namespace HeuristicLab.BioBoost.ProblemDescription {

  [StorableClass]
  public class BioBoostProblemData : ParameterizedNamedItem {

    #region Events
    public event EventHandler GeometryChanged;
    protected void OnGeometryChanged() {
      var h = GeometryChanged;
      if (h != null) h(this, EventArgs.Empty);
    }
    #endregion

    public new ParameterCollection Parameters {
      get { return base.Parameters; }
    }

    // this prevents geometries from being checked by tree traversal
    private readonly Closure<GeometryFeatureProvider> geometry = new Closure<GeometryFeatureProvider>(Closure<GeometryFeatureProvider>.Encapsulation.Referenced);

    [Storable]
    public GeometryFeatureProvider Geometry {
      get { return geometry.Value; }
      private set { geometry.Value = value; }
    }

    #region IStorableContent Members
    public string Filename { get; set; }
    #endregion

    #region Parameters
    public ValueParameter<StringArray> LocationNamesParameter {
      get { return (ValueParameter<StringArray>)Parameters["LocationNames"]; }
    }
    public ValueParameter<NeighborhoodTable> NeighborsParameter {
      get { return (ValueParameter<NeighborhoodTable>)Parameters["Neighbors"]; }
    }
    private ValueParameter<ItemCollection<DataItem>> DataItemsParameter {
      get { return (ValueParameter<ItemCollection<DataItem>>)Parameters["DataItems"]; }
    }
    public ValueParameter<ParameterCollection> FeedstockPotentialsParameter {
      get { return (ValueParameter<ParameterCollection>)Parameters["FeedstockPotentials"]; }
    }
    public ValueParameter<DoubleArray> InvestmentCostFactorsParameter {
      get { return (ValueParameter<DoubleArray>)Parameters["InvestmentCostFactors"]; }
    }
    public ValueParameter<DoubleArray> MaintenanceCostFactorsParameter {
      get { return (ValueParameter<DoubleArray>)Parameters["MaintenanceCostFactors"]; }
    }
    public ValueParameter<DoubleArray> FuelCostFactorsParameter {
      get { return (ValueParameter<DoubleArray>)Parameters["FuelCostFactors"]; }
    }
    public ValueParameter<DoubleArray> LaborCostFactorsParameter {
      get { return (ValueParameter<DoubleArray>)Parameters["LaborCostFactors"]; }
    }
    public ValueParameter<DoubleArray> OtherCostFactorsParameter {
      get { return (ValueParameter<DoubleArray>)Parameters["OtherCostFactors"]; }
    }
    public ValueParameter<CheckedItemList<StringValue>> UtilizationsParameter {
      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["Utilizations"]; }
    }
    public ValueParameter<CheckedItemList<StringValue>> TransportTargetsParameter {
      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["TransportTargets"]; }
    }
    public ValueParameter<CheckedItemList<StringValue>> FinalProductsParameter {
      get { return (ValueParameter<CheckedItemList<StringValue>>)Parameters["FinalProducts"]; }
    }
    public IValueParameter<DoubleMatrix> SplitRegionDistanceFactorsParameter {
      get { return (IValueParameter<DoubleMatrix>) Parameters["SplitRegionDistanceFactors"]; }
    }
    public IValueParameter<StringMatrix> ProductLinksParameter { get { return (IValueParameter<StringMatrix>) Parameters["ProductLinks"]; } }
    #endregion

    #region Parameter Values
    public StringArray LocationNames {
      get { return LocationNamesParameter.Value; }
      set { LocationNamesParameter.Value = value; }
    }
    public NeighborhoodTable Neighbors {
      get { return NeighborsParameter.Value; }
      set { NeighborsParameter.Value = value; }
    }
    public ItemCollection<DataItem> DataItems {
      get { return DataItemsParameter.Value; }
      private set {
        DataItemsParameter.Value = value;
        Products = value.OfType<Product>().ToList();
        Conversions = value.OfType<Conversion>().ToLookup(c => c.Feedstock, c => c);
        Logistics = value.OfType<Logistic>().ToLookup(l => l.Product, l => l);
      }
    }
    public ParameterCollection FeedstockPotentials {
      get { return FeedstockPotentialsParameter.Value; }
      set { FeedstockPotentialsParameter.Value = value; }
    }
    public DoubleArray InvestmentCostFactors {
      get { return InvestmentCostFactorsParameter.Value; }
      set { InvestmentCostFactorsParameter.Value = value; }
    }
    public DoubleArray MaintenanceCostFactors {
      get { return MaintenanceCostFactorsParameter.Value; }
      set { MaintenanceCostFactorsParameter.Value = value; }
    }
    public DoubleArray FuelCostFactors {
      get { return FuelCostFactorsParameter.Value; }
      set { FuelCostFactorsParameter.Value = value; }
    }
    public DoubleArray LaborCostFactors {
      get { return LaborCostFactorsParameter.Value; }
      set { LaborCostFactorsParameter.Value = value; }
    }
    public DoubleArray OtherCostFactors {
      get { return OtherCostFactorsParameter.Value; }
      set { OtherCostFactorsParameter.Value = value; }
    }
    public CheckedItemList<StringValue> Utilizations {
      get { return UtilizationsParameter.Value; }
      set { UtilizationsParameter.Value = value; }
    }
    public CheckedItemList<StringValue> TransportTargets {
      get { return TransportTargetsParameter.Value; }
      set { TransportTargetsParameter.Value = value; }
    }
    public CheckedItemList<StringValue> FinalProducts {
      get { return FinalProductsParameter.Value; }
      set { FinalProductsParameter.Value = value; }
    }
    public DoubleMatrix SplitRegionDistanceFactors {
      get { return SplitRegionDistanceFactorsParameter.Value; }
      set { SplitRegionDistanceFactorsParameter.Value = value; }
    }
    public IEnumerable<Product> Products { get; private set; }
    public ILookup<string, Conversion> Conversions { get; private set; }
    public ILookup<string, Logistic> Logistics { get; private set; }
    public StringMatrix ProductLinks {
      get { return ProductLinksParameter.Value; }
      set { ProductLinksParameter.Value = value; }
    }
    #endregion

    #region Construction & Cloning

    [StorableConstructor]
    protected BioBoostProblemData(bool isDeserializing) : base(isDeserializing) { }
    protected BioBoostProblemData(BioBoostProblemData orig, Cloner cloner)
      : base(orig, cloner) {
      Geometry = orig.Geometry;
      DataItems = DataItems; // invoke setter to populate Products, Conversions and Logistics
    }
    public BioBoostProblemData() {
      Parameters.Add(new ValueParameter<StringArray>("LocationNames", "The names of the individual locations."));
      Parameters.Add(new ValueParameter<NeighborhoodTable>("Neighbors", "The table of neighborhood degrees."));
      Parameters.Add(new ValueParameter<ItemCollection<DataItem>>("DataItems", "Data descriptors, e.g. for products or logistics and conversion processes."));
      Parameters.Add(new ValueParameter<ParameterCollection>("FeedstockPotentials", "Imported feedstock potentials."));
      Parameters.Add(new ValueParameter<DoubleArray>("InvestmentCostFactors", "The scaling factors for investment costs."));
      Parameters.Add(new ValueParameter<DoubleArray>("MaintenanceCostFactors", "The scaling factors for maintenance costs."));
      Parameters.Add(new ValueParameter<DoubleArray>("FuelCostFactors", "The scaling factors for fuel costs."));
      Parameters.Add(new ValueParameter<DoubleArray>("LaborCostFactors", "The scaling factors for labor costs."));
      Parameters.Add(new ValueParameter<DoubleArray>("OtherCostFactors", "The scaling factors for other costs."));
      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("Utilizations", "The possible utilization vectors."));
      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("TransportTargets", "The possible transport target vectors."));
      Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("FinalProducts", "The list of possible final products used for quality evaluation."));
      Parameters.Add(new ValueParameter<DoubleMatrix>("SplitRegionDistanceFactors", "Describes the reduction in distance if the region would be split into different numbers of smaller regions"));
      Parameters.Add(new ValueParameter<StringMatrix>("ProductLinks", "List of tuples of product names that are linked in the logistic chain"));
    }
    public override IDeepCloneable Clone(Cloner cloner) {
      return new BioBoostProblemData(this, cloner);
    }
    [StorableHook(HookType.AfterDeserialization)]
    public void AfterDeserialization() {
      DataItems = DataItems; // invoke setter to populate Products, Conversions and Logistics
      TryUpgradeStringArrayToCheckedItemList("Utilizations");
      TryUpgradeStringArrayToCheckedItemList("TransportTargets");
      if (!Parameters.ContainsKey("SplitRegionDistanceFactors"))
        Parameters.Add(new ValueParameter<DoubleMatrix>("SplitRegionDistanceFactors", "Describes the reduction in distance if the region would be split into different numbers of smaller regions"));
      if (!Parameters.ContainsKey("ProductLinks"))
        Parameters.Add(new ValueParameter<StringMatrix>("ProductLinks", "List of tuples of product names that are linked in the logistic chain"));
      if (!Parameters.ContainsKey("FinalProducts"))
        Parameters.Add(new ValueParameter<CheckedItemList<StringValue>>("FinalProducts", "The list of possible final products used for quality evaluation."));
    }

    private void TryUpgradeStringArrayToCheckedItemList(string parameterName) {
      var oldParameter = Parameters[parameterName] as ValueParameter<StringArray>;
      if (oldParameter != null) {
        var newParameter = new ValueParameter<CheckedItemList<StringValue>>(oldParameter.Name, oldParameter.Description);
        Parameters.Remove(parameterName);
        foreach (var s in oldParameter.Value)
          newParameter.Value.Add(new StringValue(s), true);
        Parameters.Add(newParameter);
      }
    }
    #endregion

    public override void CollectParameterValues(IDictionary<string, IItem> values) {
      base.CollectParameterValues(values);
      foreach (var di in DataItems) {
        di.CollectParameterValues(values);
      }
    }

    #region Data Parsing & Import
    private void AddParameter<TValue>(string name, TValue value) where TValue : class, IItem {
      Parameters.Remove(name);
      Parameters.Add(new ValueParameter<TValue>(name, value));
    }

    public void LoadFeedstockPotentials(string filename) {
      var feedstockPotentials = new List<Tuple<string, List<double>>>();
      var regionNames = new List<string>();
      bool headerDone = false;
      foreach (var line in File.ReadLines(filename)) {
        var fields = line.Split(';');
        if (!headerDone) {
          feedstockPotentials.AddRange(
            fields.Skip(1).Select(feedstockName => Tuple.Create(feedstockName, new List<double>())));
          headerDone = true;
        } else {
          regionNames.Add(fields[0]);
          for (int i = 1; i < fields.Length; i++) {
            double value;
            double.TryParse(fields[i], NumberStyles.Float, NumberFormatInfo.InvariantInfo, out value);
            feedstockPotentials[i - 1].Item2.Add(value);
          }
        }
      }

      if (LocationNames.Length > 0) {
        var newRegionNames = new HashSet<string>(regionNames);
        newRegionNames.ExceptWith(LocationNames);
        var indexTable = LocationNames.Concat(newRegionNames)
          .Select((n, i) => new { idx = i, name = n })
          .ToDictionary(p => p.name, p => p.idx);
        if (newRegionNames.Count > 0)
          LocationNames = new StringArray(LocationNames.Concat(newRegionNames).ToArray());
        foreach (var fp in feedstockPotentials) {
          var potentials = new DoubleArray(indexTable.Count);
          var potentialName = LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(fp.Item1);
          for (int i = 0; i < regionNames.Count; i++) {
            potentials[indexTable[regionNames[i]]] = fp.Item2[i];
          }
          FeedstockPotentials.Remove(potentialName);
          FeedstockPotentials.Add(new ValueParameter<DoubleArray>(potentialName, potentials));
        }
      } else {
        LocationNames = new StringArray(regionNames.ToArray());
        foreach (var fp in feedstockPotentials) {
          FeedstockPotentials.Add(new ValueParameter<DoubleArray>(LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(fp.Item1), new DoubleArray(fp.Item2.ToArray())));
        }
      }


      Configure();
    }

    public void LoadShapeFile(string filename) {
      Geometry = ShapeFileLoader.LoadShapeFile(filename);
      OnGeometryChanged();
    }

    public void AddDoubleArrayParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
      int i = 0;
      foreach (var f in fields) {
        if (i >= names.Count) break;
        var subfields = f.Split('/');
        var values = new List<double>();
        foreach (var sf in subfields) {
          double d;
          Double.TryParse(sf, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
          values.Add(d);
        }
        AddParameter(names[i] + prefix, new DoubleArray(values.ToArray()));
        i++;
      }
    }

    public void AddDoubleValueParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
      int i = 0;
      foreach (var f in fields) {
        if (i >= names.Count) break;
        double d;
        Double.TryParse(f, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
        AddParameter(names[i] + prefix, new DoubleValue(d));
        i++;
      }
    }

    public void AddIntValueParameters(IEnumerable<string> fields, IList<string> names, string prefix) {
      int i = 0;
      foreach (var f in fields) {
        if (i >= names.Count) break;
        int n;
        Int32.TryParse(f, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out n);
        AddParameter(names[i] + prefix, new IntValue(n));
        i++;
      }
    }

    public void LoadDistanceMatrix(string filename) {
      var newLocationNames = new HashSet<string>();
      var tuples = new List<Tuple<string, string, double>>();
      var matrixName = "";
      foreach (var line in File.ReadLines(filename)) {
        var l = line.Trim();
        if (l.StartsWith("#")) {
          matrixName = l.TrimStart("# ".ToCharArray());
        } else {
          var fields = l.Split(';').ToList();
          newLocationNames.Add(fields[0]);
          newLocationNames.Add(fields[1]);
          double d;
          Double.TryParse(fields[2], NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d);
          tuples.Add(Tuple.Create(fields[0], fields[1], d));
        }
      }
      newLocationNames.ExceptWith(LocationNames);
      if (newLocationNames.Count > 0) {
        LocationNames = new StringArray(LocationNames.Concat(newLocationNames).ToArray());
        foreach (ValueParameter<DoubleArray> p in FeedstockPotentials) {
          p.Value = new DoubleArray(p.Value.Concat(new double[newLocationNames.Count]).ToArray());
        }
      }
      var dm = new DistanceMatrix();
      dm.Load(tuples, LocationNames, -1);
      matrixName += "DistanceMatrix";
      if (Parameters.ContainsKey(matrixName))
        Parameters.Remove(matrixName);
      Parameters.Add(new ValueParameter<DistanceMatrix>(matrixName, dm));
      Configure();
    }

    public void LoadSplitRegionDistanceFactors(string filename) {
      // format: regionName;factor2;factor3;factor4;...
      var newLocationNames = new HashSet<string>();
      var splits = new Dictionary<string, List<double>>();
      int maxN = 0;
      foreach (var line in File.ReadLines(filename)) {
        var l = line.Trim();
        if (l.ToUpper().StartsWith("NUTS")) continue;
        var fields = line.Split(';').ToList();
        newLocationNames.Add(fields[0]);
        splits.Add(fields[0], fields.Skip(1).Select(s => ParseDouble(s, 1)).ToList());
        maxN = Math.Max(fields.Count, maxN);
      }
      newLocationNames.ExceptWith(LocationNames);
      if (newLocationNames.Count > 0) {
        LocationNames = new StringArray(LocationNames.Concat(newLocationNames).ToArray());
        SplitRegionDistanceFactors = new DoubleMatrix(LocationNames.Length, maxN - 1, 
          Enumerable.Range(2, maxN - 1).Select(d => d.ToString(NumberFormatInfo.InvariantInfo)),
          LocationNames);
        for (int row = 0; row < SplitRegionDistanceFactors.Rows; row++) {
          List<double> values;
          if (splits.TryGetValue(LocationNames[row], out values)) {
            for (int col = 0; col < values.Count; col++) {
              SplitRegionDistanceFactors[row, col] = values[col];
            }
          }
        }
      }
    }

    public static double ParseDouble(string s, double defaultValue = 0) {
      double d;
      if (Double.TryParse(s, NumberStyles.Float, NumberFormatInfo.InvariantInfo, out d))
        return d;
      return defaultValue;
    }

    public void RemoveRegions(Regex regex) {
      RemoveRegions(LocationNames.Where(n => regex.IsMatch(n)));
    }

    public void KeepRegions(Regex regex) {
      RemoveRegions(LocationNames.Where(n => !regex.IsMatch(n)));
    }

    public void RemoveRegions(IEnumerable<string> regionNames) {
      var regionNameSet = new HashSet<string>(regionNames);
      var removedIndices =
        LocationNames.Select((r, i) => new { r, i })
                     .Where(ri => regionNameSet.Contains(ri.r))
                     .Select(ri => ri.i)
                     .ToList();
      // remove region names
      int nLocationNamesBefore = LocationNames.Length;
      LocationNames = new StringArray(LocationNames.RemoveAtIndices(removedIndices).ToArray());
      int nLocationNamesAfter = LocationNames.Length;
      Debug.Assert(nLocationNamesBefore - removedIndices.Count == nLocationNamesAfter);
      // remove region potentials
      foreach (var potentialName in FeedstockPotentials.Select(p => p.Name).ToList()) {
        var potentials = FeedstockPotentials[potentialName];
        FeedstockPotentials.Remove(potentialName);
        FeedstockPotentials.Add(
          new ValueParameter<DoubleArray>(
            potentialName,
            new DoubleArray(((ValueParameter<DoubleArray>)potentials).Value.RemoveAtIndices(removedIndices).ToArray())));
      }
      // remove region distances
      foreach (var distanceMatrixParam in Parameters.Where(p => p.Name.EndsWith("DistanceMatrix"))) {
        var valueParam = distanceMatrixParam as IValueParameter;
        if (valueParam != null && typeof(DistanceMatrix).IsAssignableFrom(valueParam.DataType)) {
          var matrix = (DistanceMatrix)valueParam.Value;
          valueParam.Value = matrix.CopyAndRemoveIndices(removedIndices);
        }
      }
      // remove region neighbors
      if (Neighbors != null)
        Neighbors = Neighbors.CopyAndRemoveIndices(removedIndices);
      // remove shapes & and create new geometry (copy on write)
      var locationNameset = new HashSet<string>(LocationNames);
      var keptRows = Geometry.Features.Cast<FeatureDataRow>()
        .Where(row => locationNameset.Contains(row["NUTS_ID"]))
        .ToList();
      var newFeatures = new FeatureDataTable();
      foreach (DataColumn column in Geometry.Features.Columns) {
        newFeatures.Columns.Add(column.ColumnName, column.DataType);
      }
      foreach (FeatureDataRow row in keptRows) {
        var newRow = (FeatureDataRow)newFeatures.LoadDataRow(row.ItemArray, LoadOption.OverwriteChanges);
        newRow.Geometry = (IGeometry) row.Geometry.Clone();
      }
      newFeatures.AcceptChanges();
      Geometry = new GeometryFeatureProvider(newFeatures);
      OnGeometryChanged();
      Configure();
    }

    public void LoadNeighbors(string filename) {
      var locationNames = LocationNames.Select((n, i) => new { n, i }).ToDictionary(p => p.n, p => p.i);
      var neighbors = new NeighborhoodTable();
      foreach (var line in File.ReadLines(filename)) {
        var fields = line.Split(';');
        int id1, id2, degree;
        if (locationNames.TryGetValue(fields[0], out id1) &&
            locationNames.TryGetValue(fields[1], out id2) &&
            Int32.TryParse(fields[2], out degree))
          neighbors.AddEntry(id1, id2, degree);
      }
      neighbors.Check();
      Neighbors = neighbors;
    }

    public void LoadDescriptors(string filename) {
      var newItems = new ItemCollection<DataItem>(DataItem.ParseFile(filename));
      if (DataItems != null) {
        foreach (var oldItem in DataItems) {
          if (!newItems.Any(i => i.IsEquivalentTo(oldItem)))
            newItems.Add(oldItem);
        }
      }
      DataItems = newItems;
      Configure();
    }

    private void Configure() {
      DistributeCostFactors();
      DiscoverSolutionSpace();
    }

    private void DistributeCostFactors() {
      int length = LocationNames.Length;
      InvestmentCostFactors = new DoubleArray(length);
      MaintenanceCostFactors = new DoubleArray(length);
      FuelCostFactors = new DoubleArray(length);
      LaborCostFactors = new DoubleArray(length);
      OtherCostFactors = new DoubleArray(length);
      var costFactorDescriptors = DataItems.OfType<CostFactors>().FirstOrDefault();
      if (costFactorDescriptors == null) return;
      for (int i = 0; i < length; i++) {
        var countryCode = LocationNames[i].Substring(0, 2);
        InvestmentCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Investment, countryCode, 1);
        MaintenanceCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Maintenance, countryCode, 1);
        FuelCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Fuel, countryCode, 1);
        LaborCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Labor, countryCode, 1);
        OtherCostFactors[i] = GetDoubleValueValue(costFactorDescriptors.Other, countryCode, 1);
      }
    }

    private void DiscoverSolutionSpace() {
      if (DataItems == null || DataItems.Count == 0) return;
      var products = Products;
      var logistics = Logistics;
      var conversions = DataItems.OfType<Conversion>().ToList();

      // perform conceptual logistic simulation
      var usableProducts = products
        .Where(product => FeedstockPotentials.ContainsKey(LayerDescriptor.PotentialsFromProblemData.NameWithPrefix(product.Label)))
        .ToList();
      var transportableProducts = usableProducts
        .Where(p => logistics.Contains(p.Label))
        .ToList();
      var feasibleDecentralConversions = conversions
        .Where(c => transportableProducts.Any(p => p.Label == c.Feedstock))
        .ToList();
      var possibleIntermediates = feasibleDecentralConversions
        .Aggregate(Enumerable.Empty<string>(),
                   (intermediates, conv) => intermediates.Concat(conv.Products.Select(p => p.Name)))
        .ToList();
      var transportableIntermediates = possibleIntermediates
        .Where(logistics.Contains) // && intermediate exists as product?
        .ToList();
      var feasibleCentralConversions = conversions
        .Where(c => transportableIntermediates.Any(i => i == c.Feedstock))
        .ToList();
      var finalProductLists = feasibleCentralConversions
        .Select(c => c.Products.Where(p => { var v = (p.Value as DoubleValue); return v != null && v.Value > 0; }).Select(p => p.Name));
      var finalProductList = finalProductLists.Aggregate(Enumerable.Empty<string>(), (x, y) => x.Concat(y));
      var uniqueFinalProducts = finalProductList.GroupBy(p => p).Select(p => p.Key);
      var valuableFinalProducts = uniqueFinalProducts.Where(productName => Products.Any(p => p.Label == productName && p.Price > 0));
      var finalProducts = valuableFinalProducts.ToList();

      // TODO: forward simulation sanity checks
      // non-sold products
      // non-(transported/converted/sold) intermediates
      // non-transported/converted feedstocks

      // reverse pruning
      // select only central conversions for which valuable end-products exist
      var necessaryCentralConversions = feasibleCentralConversions
        .Where(c => c.Products.Any(p => products.Any(p1 => p1.Label == p.Name && p1.Price > 0)))
        .ToList();
      // transport only intermediates that need to be converted
      var neccessaryIntermediates = transportableIntermediates
        .Where(i => necessaryCentralConversions.Any(c => c.Feedstock == i))
        .ToList();
      // select only intermediates that either
      // - need to transported and are further processed or
      // - can be sold
      var feasibleIntermediates = neccessaryIntermediates.Concat(
        possibleIntermediates.Where(i => products.Any(p => p.Label == i && p.Price > 0)))
        .ToList();
      // produce only selected intermediates
      var necessaryDecentralConversions = feasibleDecentralConversions
        .Where(c => c.Products.Any(p => feasibleIntermediates.Contains(p.Name)))
        .ToList();
      // transport and buy only feedstocks that are converted and sold
      var necessaryFeedstocks = necessaryDecentralConversions
        .Select(c => c.Feedstock).ToList();

      // enable feedstocks directly to central conversion? Yes, it automatically becomes a decentral conversion
      // TODO: allow more than two echelons?

      // configure search space:
      UpdateSearchSpace(Utilizations, necessaryFeedstocks);
      UpdateSearchSpace(TransportTargets, necessaryFeedstocks.Concat(neccessaryIntermediates));
      UpdateSearchSpace(FinalProducts, finalProducts);

      var q =
        (from p1 in necessaryFeedstocks.Concat(neccessaryIntermediates)
          from p2 in neccessaryIntermediates
          where conversions.Any(c => c.Feedstock == p1 && c.Products.ContainsKey(p2))
          select new {p1, p2}).Concat(
        from p1 in neccessaryIntermediates
        from p2 in finalProducts
        where conversions.Any(c => c.Feedstock == p1 && c.Products.ContainsKey(p2))
        select new {p1, p2})
       .Distinct().ToList();
      var productLinks = new StringMatrix(q.Count, 2);
      for (int i = 0; i < q.Count; i++) {
        productLinks[i, 0] = q[i].p1;
        productLinks[i, 1] = q[i].p2;
      }
      ProductLinks = productLinks;

    }

    private void UpdateSearchSpace(CheckedItemList<StringValue> choices, IEnumerable<string> requirements) {
      var req = new HashSet<string>(requirements);
      for (int i = 0; i < choices.Count; i++) {
        if (req.Contains(choices[i].Value)) {
          req.Remove(choices[i].Value); // already there, not required anymore
        } else {
          choices.SetItemCheckedState(i, false);
        }
      }
      foreach (var r in req) {
        choices.Add(new StringValue(r), true);
      }
    }

    private static double GetDoubleValueValue(ValueParameterCollection parameters, string name, double defaultValue) {
      IValueParameter param;
      if (!parameters.TryGetValue(name, out param)) return defaultValue;
      var doubleValue = param.Value as DoubleValue;
      if (doubleValue == null) return defaultValue;
      return doubleValue.Value;
    }

    #endregion

    #region Data Consistency Checks
    public string CheckGeoData() {
      var warnings = new List<string>();
      CheckDistanceMatrices(warnings);
      var shapeNames = CheckShapeNames(warnings);
      var locationNames = CheckLocationNames(warnings);
      FindMissing(warnings, locationNames, shapeNames, "locations without shapes");
      FindMissing(warnings, shapeNames, locationNames, "shapes without data");
      var totalPotentials = new double[LocationNames.Length];
      foreach (var p in FeedstockPotentials.OfType<ValueParameter<DoubleArray>>()) {
        totalPotentials = totalPotentials.Zip(p.Value, (a, b) => a + b).ToArray();
        FindZeros(warnings, p.Value, p.Name);
      }
      FindZeros(warnings, totalPotentials, "any potential");
      return string.Join("\n\n", warnings);
    }

    private void FindZeros(List<string> warnings, IEnumerable<double> values, string name) {
      var zeros = values.Select((v, i) => new { v, i }).Where(p => p.v == 0).Select(p => LocationNames[p.i]).ToList();
      if (zeros.Count > 0)
        warnings.Add(string.Format("{0} regions with zero of {1}:\n  {2}",
          zeros.Count, name, string.Join(", ", zeros)));
    }

    private static void FindMissing(List<string> warnings, IEnumerable<string> items, IEnumerable<string> other, string message) {
      var missingItems = new HashSet<string>(items);
      missingItems.ExceptWith(other);
      if (missingItems.Count > 0)
        warnings.Add(string.Format("{0} {1}:\n  {2}",
          missingItems.Count, message, string.Join(", ", missingItems)));
    }

    private HashSet<string> CheckLocationNames(List<string> warnings) {
      var duplicates = new List<String>();
      var locationNames = new HashSet<string>();
      foreach (var n in LocationNames) {
        if (locationNames.Contains(n))
          duplicates.Add(n);
        locationNames.Add(n);
      }
      if (duplicates.Count > 0)
        warnings.Add(string.Format("{0} duplicate location names:\n  {1}",
          duplicates.Count, string.Join(", ", duplicates)));
      return locationNames;
    }

    private HashSet<string> CheckShapeNames(List<string> warnings) {
      var duplicates = new List<string>();
      var shapeNames = new HashSet<string>();
      foreach (FeatureDataRow row in Geometry.Features.Rows) {
        var n = (string)row["NUTS_ID"];
        if (shapeNames.Contains(n))
          duplicates.Add(n);
        shapeNames.Add(n);
      }
      if (duplicates.Count > 0)
        warnings.Add(string.Format("{0} duplicate shape names:\n  {1}",
          duplicates.Count, String.Join(", ", duplicates)));
      return shapeNames;
    }

    private void CheckDistanceMatrices(List<string> warnings) {
      foreach (var distanceMatrixParam in Parameters.Where(p => p.Name.EndsWith("DistanceMatrix"))) {
        var valueParam = distanceMatrixParam as IValueParameter;
        if (valueParam != null && typeof(DistanceMatrix).IsAssignableFrom(valueParam.DataType))
          FindUnconnectedRegions((DistanceMatrix)valueParam.Value, warnings);
      }
    }

    private void FindUnconnectedRegions(DistanceMatrix dm, List<string> warnings) {
      var unconnected = new List<string>();
      var hardlyConnected = new List<string>();
      for (int i = 0; i < dm.Size; i++) {
        var nConnected = 0;
        for (int j = 0; j < i; j++) {
          if (dm[i, j] > 0) nConnected++;
        }
        if (nConnected == 0)
          unconnected.Add(LocationNames[i]);
        else if (1.0 * nConnected < 1.0 * LocationNames.Length / 100)
          hardlyConnected.Add(LocationNames[i]);
      }
      if (unconnected.Count > 0)
        warnings.Add(string.Format("{0} unconnected regions:\n  {1}",
          unconnected.Count, string.Join(", ", unconnected)));
      if (hardlyConnected.Count > 0)
        warnings.Add(string.Format("{0} connected to fewer than 1% of regions:\n  {1}",
          hardlyConnected.Count, string.Join(", ", hardlyConnected)));
    }

    // TODO: check consistency of conversion process
    #endregion
  }
}