Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
07/08/15 15:32:12 (9 years ago)
Author:
dglaser
Message:

#2388: Merged trunk into HiveStatistics branch

Location:
branches/HiveStatistics/sources
Files:
9 edited

Legend:

Unmodified
Added
Removed
  • branches/HiveStatistics/sources

  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4

    • Property svn:mergeinfo set to (toggle deleted branches)
      /stable/HeuristicLab.DataPreprocessing/3.4mergedeligible
      /trunk/sources/HeuristicLab.DataPreprocessing/3.4mergedeligible
      /branches/1721-RandomForestPersistence/HeuristicLab.DataPreprocessing/3.410321-10322
      /branches/1721-RandomForestPersistence/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410321-10322
      /branches/Algorithms.GradientDescent/HeuristicLab.DataPreprocessing/3.45516-5520
      /branches/Algorithms.GradientDescent/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45516-5520
      /branches/Benchmarking/sources/HeuristicLab.DataPreprocessing/3.46917-7005
      /branches/Benchmarking/sources/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46917-7005
      /branches/CloningRefactoring/HeuristicLab.DataPreprocessing/3.44656-4721
      /branches/CloningRefactoring/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.44656-4721
      /branches/CodeEditor/HeuristicLab.DataPreprocessing/3.411700-11806
      /branches/CodeEditor/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.411700-11806
      /branches/DataAnalysis Refactoring/HeuristicLab.DataPreprocessing/3.45471-5808
      /branches/DataAnalysis Refactoring/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45471-5808
      /branches/DataAnalysis SolutionEnsembles/HeuristicLab.DataPreprocessing/3.45815-6180
      /branches/DataAnalysis SolutionEnsembles/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45815-6180
      /branches/DataAnalysis/HeuristicLab.DataPreprocessing/3.44458-4459,​4462,​4464
      /branches/DataAnalysis/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.44458-4459,​4462,​4464
      /branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.410085-11101
      /branches/DataPreprocessing/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410085-11101
      /branches/DataPreprocessingImprovements/3.412051
      /branches/DataPreprocessingImprovements/HeuristicLab.DataPreprocessing/3.412053-12675
      /branches/GP.Grammar.Editor/HeuristicLab.DataPreprocessing/3.46284-6795
      /branches/GP.Grammar.Editor/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46284-6795
      /branches/GP.Symbols (TimeLag, Diff, Integral)/HeuristicLab.DataPreprocessing/3.45060
      /branches/GP.Symbols (TimeLag, Diff, Integral)/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45060
      /branches/HLScript/HeuristicLab.DataPreprocessing/3.410331-10358
      /branches/HLScript/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410331-10358
      /branches/HeuristicLab.DatasetRefactor/sources/HeuristicLab.DataPreprocessing/3.411570-12508
      /branches/HeuristicLab.Problems.DataAnalysis.Trading/HeuristicLab.DataPreprocessing/3.46123-9799
      /branches/HeuristicLab.Problems.DataAnalysis.Trading/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46123-9799
      /branches/LogResidualEvaluator/HeuristicLab.DataPreprocessing/3.410202-10483
      /branches/LogResidualEvaluator/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410202-10483
      /branches/NET40/sources/HeuristicLab.DataPreprocessing/3.45138-5162
      /branches/NET40/sources/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45138-5162
      /branches/NSGA-II Changes/HeuristicLab.DataPreprocessing/3.412033-12122
      /branches/ParallelEngine/HeuristicLab.DataPreprocessing/3.45175-5192
      /branches/ParallelEngine/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45175-5192
      /branches/ProblemInstancesRegressionAndClassification/HeuristicLab.DataPreprocessing/3.47568-7810
      /branches/ProblemInstancesRegressionAndClassification/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.47568-7810
      /branches/QAPAlgorithms/HeuristicLab.DataPreprocessing/3.46350-6627
      /branches/QAPAlgorithms/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46350-6627
      /branches/Restructure trunk solution/HeuristicLab.DataPreprocessing/3.46828
      /branches/Restructure trunk solution/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46828
      /branches/RuntimeOptimizer/HeuristicLab.DataPreprocessing/3.48943-9078
      /branches/RuntimeOptimizer/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.48943-9078
      /branches/ScatterSearch (trunk integration)/HeuristicLab.DataPreprocessing/3.47787-8333
      /branches/ScatterSearch (trunk integration)/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.47787-8333
      /branches/SlaveShutdown/HeuristicLab.DataPreprocessing/3.48944-8956
      /branches/SlaveShutdown/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.48944-8956
      /branches/SpectralKernelForGaussianProcesses/HeuristicLab.DataPreprocessing/3.410204-10479
      /branches/SpectralKernelForGaussianProcesses/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410204-10479
      /branches/SuccessProgressAnalysis/HeuristicLab.DataPreprocessing/3.45370-5682
      /branches/SuccessProgressAnalysis/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45370-5682
      /branches/Trunk/HeuristicLab.DataPreprocessing/3.46829-6865
      /branches/Trunk/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.46829-6865
      /branches/UnloadJobs/HeuristicLab.DataPreprocessing/3.49168-9215
      /branches/UnloadJobs/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.49168-9215
      /branches/VNS/HeuristicLab.DataPreprocessing/3.45594-5752
      /branches/VNS/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45594-5752
      /branches/histogram/HeuristicLab.DataPreprocessing/3.45959-6341
      /branches/histogram/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.45959-6341
      /stable/HeuristicLab.DataPreprocessing/HeuristicLab.DataPreprocessing/3.410032-10033,​11170,​11173,​11175,​12008-12010
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/DataGridContent.cs

    r12012 r12689  
    114114    }
    115115
    116     public void DeleteRow(IEnumerable<int> rows) {
     116    public void DeleteRows(IEnumerable<int> rows) {
    117117      PreProcessingData.DeleteRowsWithIndices(rows);
    118118      createRowNames();
     119    }
     120
     121    public void DeleteColumn(int column) {
     122      PreProcessingData.DeleteColumn(column);
    119123    }
    120124
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/HistogramContent.cs

    r12012 r12689  
    4242      set { this.classifierVariableIndex = value; }
    4343    }
     44    public bool IsDetailedChartViewEnabled { get; set; }
    4445
    4546
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/PreprocessingChartContent.cs

    r12012 r12689  
    2323using System.Collections.Generic;
    2424using System.Drawing;
     25using System.Linq;
    2526using HeuristicLab.Analysis;
    2627using HeuristicLab.Common;
     
    126127
    127128
    128     public ICheckedItemList<StringValue> CreateVariableItemList() {
     129    public ICheckedItemList<StringValue> CreateVariableItemList(IEnumerable<string> checkedItems = null) {
    129130      ICheckedItemList<StringValue> itemList = new CheckedItemList<StringValue>();
    130131      foreach (string name in PreprocessingData.GetDoubleVariableNames()) {
    131         itemList.Add(new StringValue(name), true);
     132        var n = new StringValue(name);
     133        itemList.Add(n, (checkedItems == null) ? true : checkedItems.Contains(name));
    132134      }
    133135      return new ReadOnlyCheckedItemList<StringValue>(itemList);
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/SearchLogic.cs

    r12012 r12689  
    2323using System.Collections;
    2424using System.Collections.Generic;
    25 using System.Linq;
    2625
    2726namespace HeuristicLab.DataPreprocessing {
    2827  public class SearchLogic : ISearchLogic {
    2928    private readonly ITransactionalPreprocessingData preprocessingData;
     29    private readonly IFilterLogic filterLogic;
    3030
    3131    private Dictionary<int, IList<int>> MissingValueIndicies { get; set; }
     
    4444    }
    4545
    46     public SearchLogic(ITransactionalPreprocessingData thePreprocessingData) {
     46    public SearchLogic(ITransactionalPreprocessingData thePreprocessingData, IFilterLogic theFilterLogic) {
    4747      preprocessingData = thePreprocessingData;
     48      filterLogic = theFilterLogic;
    4849
    4950      MissingValueIndicies = new Dictionary<int, IList<int>>();
    5051      ValuesWithoutNaN = new Dictionary<int, IList>();
    5152
    52       preprocessingData.Changed += preprocessingData_Changed;
     53      preprocessingData.Changed += PreprocessingData_Changed;
     54      filterLogic.FilterChanged += FilterLogic_FilterChanged;
    5355    }
    5456
    55     void preprocessingData_Changed(object sender, DataPreprocessingChangedEventArgs e)
    56     {
     57    void FilterLogic_FilterChanged(object sender, EventArgs e) {
     58      //recalculate
     59      for (int i = 0; i < Columns; i++) {
     60        MissingValueIndicies.Remove(i);
     61        ValuesWithoutNaN.Remove(i);
     62      }
     63    }
     64
     65    void PreprocessingData_Changed(object sender, DataPreprocessingChangedEventArgs e) {
    5766      switch (e.Type) {
    5867        case DataPreprocessingChangedEventType.DeleteColumn:
     
    7382          ValuesWithoutNaN = new Dictionary<int, IList>();
    7483          break;
    75       } 
     84      }
    7685    }
    7786
     
    97106
    98107    public IList<int> GetMissingValueIndices(int columnIndex) {
    99       if (!MissingValueIndicies.ContainsKey(columnIndex)){       
    100           if (preprocessingData.VariableHasType<double>(columnIndex)) {
    101             MissingValueIndicies[columnIndex] = GetMissingValueIndices<double>(columnIndex);
    102           } else if (preprocessingData.VariableHasType<string>(columnIndex)) {
    103             MissingValueIndicies[columnIndex] = GetMissingValueIndices<string>(columnIndex);
    104           } else if (preprocessingData.VariableHasType<DateTime>(columnIndex)) {
    105             MissingValueIndicies[columnIndex] = GetMissingValueIndices<DateTime>(columnIndex);
    106           } else {
    107             throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
    108           }
    109       }
     108      if (!MissingValueIndicies.ContainsKey(columnIndex)) {
     109        if (preprocessingData.VariableHasType<double>(columnIndex)) {
     110          MissingValueIndicies[columnIndex] = GetMissingValueIndices<double>(columnIndex);
     111        } else if (preprocessingData.VariableHasType<string>(columnIndex)) {
     112          MissingValueIndicies[columnIndex] = GetMissingValueIndices<string>(columnIndex);
     113        } else if (preprocessingData.VariableHasType<DateTime>(columnIndex)) {
     114          MissingValueIndicies[columnIndex] = GetMissingValueIndices<DateTime>(columnIndex);
     115        } else {
     116          throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
     117        }
     118      }
     119      return MissingValueIndicies[columnIndex];
     120    }
    110121
    111       return MissingValueIndicies[columnIndex];
    112    }
    113122    private IList<int> GetMissingValueIndices<T>(int columnIndex) {
    114123      List<int> missingIndices = new List<int>();
    115      
    116       for(int row = 0; row < preprocessingData.Rows; ++row) {
     124
     125      for (int row = 0; row < preprocessingData.Rows; ++row) {
    117126        if (IsMissingValue(columnIndex, row)) {
    118127          missingIndices.Add(row);
     
    123132    }
    124133
    125     public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection)
    126     {
    127       if (considerSelection) {     
    128         var selectedRows =  preprocessingData.Selection[columnIndex];
    129        
     134    public IEnumerable<T> GetValuesWithoutNaN<T>(int columnIndex, bool considerSelection) {
     135      if (considerSelection) {
     136        var selectedRows = preprocessingData.Selection[columnIndex];
     137
    130138        List<T> values = new List<T>();
    131139        foreach (var rowIdx in selectedRows) {
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Implementations/StatisticsLogic.cs

    r12012 r12689  
    3838
    3939    public int GetColumnCount() {
    40       return preprocessingData.Columns;
     40      return searchLogic.Columns;
    4141    }
    4242
    4343    public int GetRowCount() {
    44       return preprocessingData.Rows;
     44      return searchLogic.Rows;
    4545    }
    4646
     
    4848      int count = 0;
    4949
    50       for (int i = 0; i < preprocessingData.Columns; ++i) {
     50      for (int i = 0; i < searchLogic.Columns; ++i) {
    5151        if (preprocessingData.VariableHasType<double>(i)) {
    5252          ++count;
     
    5757
    5858    public int GetNominalColumnCount() {
    59       return preprocessingData.Columns - GetNumericColumnCount();
     59      return searchLogic.Columns - GetNumericColumnCount();
    6060    }
    6161
    6262    public int GetMissingValueCount() {
    6363      int count = 0;
    64       for (int i = 0; i < preprocessingData.Columns; ++i) {
     64      for (int i = 0; i < searchLogic.Columns; ++i) {
    6565        count += GetMissingValueCount(i);
    6666      }
     
    7373
    7474    public T GetMin<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
    75       return preprocessingData.GetValues<T>(columnIndex, considerSelection).Min();
     75      var min = default(T);
     76      if (preprocessingData.VariableHasType<T>(columnIndex)) {
     77        var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
     78        if (values.Any()) {
     79          min = values.Min();
     80        }
     81      }
     82      return min;
    7683    }
    7784
    7885    public T GetMax<T>(int columnIndex, bool considerSelection) where T : IComparable<T> {
    79       return preprocessingData.GetValues<T>(columnIndex, considerSelection).Max();
     86      var max = default(T);
     87      if (preprocessingData.VariableHasType<T>(columnIndex)) {
     88        var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
     89        if (values.Any()) {
     90          max = values.Max();
     91        }
     92      }
     93      return max;
    8094    }
    8195
     
    8397      double median = double.NaN;
    8498      if (preprocessingData.VariableHasType<double>(columnIndex)) {
    85         median = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Median();
     99        var values = GetValuesWithoutNaN<double>(columnIndex, considerSelection);
     100        if (values.Any()) {
     101          median = values.Median();
     102        }
    86103      }
    87104      return median;
     
    91108      double avg = double.NaN;
    92109      if (preprocessingData.VariableHasType<double>(columnIndex)) {
    93         avg = GetValuesWithoutNaN<double>(columnIndex, considerSelection).Average();
     110        var values = GetValuesWithoutNaN<double>(columnIndex, considerSelection);
     111        if (values.Any()) {
     112          avg = values.Average();
     113        }
    94114      }
    95115      return avg;
     
    113133
    114134    public T GetMostCommonValue<T>(int columnIndex, bool considerSelection) {
    115       var t = preprocessingData.GetValues<T>(columnIndex, considerSelection);
    116       var t2 = t.GroupBy(x => x);
    117       var t3 = t2.Select(g => g.Key);
    118 
    119       return preprocessingData.GetValues<T>(columnIndex, considerSelection)
    120                               .GroupBy(x => x)
     135      var values = GetValuesWithoutNaN<T>(columnIndex, considerSelection);
     136      if (!values.Any())
     137        return default(T);
     138      return values.GroupBy(x => x)
    121139                              .OrderByDescending(g => g.Count())
    122140                              .Select(g => g.Key)
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/Interfaces/IDataGridContent.cs

    r12012 r12689  
    3131    IDictionary<int, IList<int>> Selection { get; set; }
    3232
    33     void DeleteRow(IEnumerable<int> rows);
     33    void DeleteRows(IEnumerable<int> rows);
     34    void DeleteColumn(int column);
    3435    bool Validate(string value, out string errorMessage, int columnIndex);
    3536
  • branches/HiveStatistics/sources/HeuristicLab.DataPreprocessing/3.4/ProblemDataCreator.cs

    r12012 r12689  
    6161      SetTrainingAndTestPartition(problemData);
    6262      // set the input variables to the correct checked state
    63       var inputVariables = problemData.InputVariables.ToDictionary(x => x.Value, x => x);
    64       foreach (var variable in oldProblemData.InputVariables) {
    65         bool @checked = oldProblemData.InputVariables.ItemChecked(variable);
    66         problemData.InputVariables.SetItemCheckedState(inputVariables[variable.Value], @checked);
     63      var inputVariables = oldProblemData.InputVariables.ToDictionary(x => x.Value, x => x);
     64      foreach (var variable in problemData.InputVariables) {
     65        bool isChecked = oldProblemData.InputVariables.ItemChecked(inputVariables[variable.Value]);
     66        problemData.InputVariables.SetItemCheckedState(variable, isChecked);
    6767      }
    6868
     
    7777
    7878    private IDataAnalysisProblemData CreateClassificationData(ClassificationProblemData oldProblemData) {
     79      // target variable must be double and must exist in the new dataset
    7980      var targetVariable = oldProblemData.TargetVariable;
    80       // target variable must be double and must exist in the new dataset
    81       return new ClassificationProblemData(ExportedDataset, GetDoubleInputVariables(targetVariable), targetVariable, Transformations);
     81      var newProblemData = new ClassificationProblemData(ExportedDataset, GetDoubleInputVariables(targetVariable), targetVariable, Transformations);
     82      newProblemData.PositiveClass = oldProblemData.PositiveClass;
     83      return newProblemData;
    8284    }
    8385
Note: See TracChangeset for help on using the changeset viewer.