Changeset 13584

Timestamp:

02/03/16 12:03:11 (5 years ago)

Author:

mkommend

Message:

#2071: Added possibility to specify the encoding when importing files with the table file parser and changed export functionality to use the system's default encoding for creating CSV files.

Location:

trunk/sources

Files:

• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/ClassificationImportTypeDialog.Designer.cs (modified) (3 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.Designer.cs (modified) (7 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.cs (modified) (5 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/RegressionImportTypeDialog.Designer.cs (modified) (1 diff)
• HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/TimeSeriesPrognosisImportTypeDialog.Designer.cs (modified) (5 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/DataAnalysisInstanceProvider.cs (modified) (3 diffs)
• HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs (modified) (7 diffs)

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/ClassificationImportTypeDialog.Designer.cs

@@ -87 +87 @@
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TargetVariableComboBox);
       this.ProblemDataSettingsGroupBox.Controls.Add(this.UniformDistributionOfClassesCheckbox);
-      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 312);
+      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 285);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.UniformDistributionOfClassesCheckbox, 0);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.TargetVariableComboBox, 0);
@@ -110 +110 @@
       // 
       this.PreviewDatasetMatrix.Location = new System.Drawing.Point(6, 134);
-      this.PreviewDatasetMatrix.Size = new System.Drawing.Size(435, 172);
+      this.PreviewDatasetMatrix.Size = new System.Drawing.Size(435, 145);
       // 
       // PreviewLabel
@@ -118 +118 @@
       // TargetVariableComboBox
       // 
-      this.TargetVariableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
+      this.TargetVariableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
             | System.Windows.Forms.AnchorStyles.Right)));
       this.TargetVariableComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.Designer.cs

@@ -63 +63 @@
       this.SeparatorComboBox = new System.Windows.Forms.ComboBox();
       this.CSVSettingsGroupBox = new System.Windows.Forms.GroupBox();
+      this.EncodingInfoLabel = new System.Windows.Forms.Label();
+      this.EncodingLabel = new System.Windows.Forms.Label();
+      this.EncodingComboBox = new System.Windows.Forms.ComboBox();
+      this.CheckboxColumnNames = new System.Windows.Forms.CheckBox();
       this.DateTimeFormatInfoLabel = new System.Windows.Forms.Label();
       this.DecimalSeparatorInfoLabel = new System.Windows.Forms.Label();
@@ -72 +76 @@
       this.PreviewDatasetMatrix = new HeuristicLab.Data.Views.StringConvertibleMatrixView();
       this.ToolTip = new System.Windows.Forms.ToolTip(this.components);
-      this.CheckboxColumnNames = new System.Windows.Forms.CheckBox();
       ((System.ComponentModel.ISupportInitialize)(this.TrainingTestTrackBar)).BeginInit();
       this.CSVSettingsGroupBox.SuspendLayout();
@@ -249 +252 @@
       this.CSVSettingsGroupBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
             | System.Windows.Forms.AnchorStyles.Right)));
+      this.CSVSettingsGroupBox.Controls.Add(this.EncodingInfoLabel);
+      this.CSVSettingsGroupBox.Controls.Add(this.EncodingLabel);
+      this.CSVSettingsGroupBox.Controls.Add(this.EncodingComboBox);
       this.CSVSettingsGroupBox.Controls.Add(this.CheckboxColumnNames);
       this.CSVSettingsGroupBox.Controls.Add(this.DateTimeFormatInfoLabel);
@@ -261 +267 @@
       this.CSVSettingsGroupBox.Location = new System.Drawing.Point(12, 32);
       this.CSVSettingsGroupBox.Name = "CSVSettingsGroupBox";
-      this.CSVSettingsGroupBox.Size = new System.Drawing.Size(447, 126);
+      this.CSVSettingsGroupBox.Size = new System.Drawing.Size(447, 153);
       this.CSVSettingsGroupBox.TabIndex = 16;
       this.CSVSettingsGroupBox.TabStop = false;
       this.CSVSettingsGroupBox.Text = "CSV Settings";
+      // 
+      // EncodingInfoLabel
+      // 
+      this.EncodingInfoLabel.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+      this.EncodingInfoLabel.Image = HeuristicLab.Common.Resources.VSImageLibrary.Information;
+      this.EncodingInfoLabel.Location = new System.Drawing.Point(421, 102);
+      this.EncodingInfoLabel.Name = "EncodingInfoLabel";
+      this.EncodingInfoLabel.Size = new System.Drawing.Size(16, 16);
+      this.EncodingInfoLabel.TabIndex = 27;
+      this.EncodingInfoLabel.Tag = "Select the encoding the file was saved with.";
+      this.ToolTip.SetToolTip(this.EncodingInfoLabel, "Select the encoding the file was saved with.");
+      this.EncodingInfoLabel.DoubleClick += new System.EventHandler(this.ControlToolTip_DoubleClick);
+      // 
+      // EncodingLabel
+      // 
+      this.EncodingLabel.AutoSize = true;
+      this.EncodingLabel.Location = new System.Drawing.Point(6, 103);
+      this.EncodingLabel.Name = "EncodingLabel";
+      this.EncodingLabel.Size = new System.Drawing.Size(52, 13);
+      this.EncodingLabel.TabIndex = 26;
+      this.EncodingLabel.Text = "Encoding";
+      // 
+      // EncodingComboBox
+      // 
+      this.EncodingComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
+            | System.Windows.Forms.AnchorStyles.Right)));
+      this.EncodingComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.EncodingComboBox.Enabled = false;
+      this.EncodingComboBox.FormattingEnabled = true;
+      this.EncodingComboBox.Location = new System.Drawing.Point(111, 100);
+      this.EncodingComboBox.Name = "EncodingComboBox";
+      this.EncodingComboBox.Size = new System.Drawing.Size(300, 21);
+      this.EncodingComboBox.TabIndex = 25;
+      this.EncodingComboBox.SelectionChangeCommitted += new System.EventHandler(this.CSVFormatComboBoxSelectionChangeCommitted);
+      // 
+      // CheckboxColumnNames
+      // 
+      this.CheckboxColumnNames.AutoSize = true;
+      this.CheckboxColumnNames.Location = new System.Drawing.Point(9, 127);
+      this.CheckboxColumnNames.Name = "CheckboxColumnNames";
+      this.CheckboxColumnNames.Size = new System.Drawing.Size(144, 17);
+      this.CheckboxColumnNames.TabIndex = 24;
+      this.CheckboxColumnNames.Text = "Column names in first line";
+      this.CheckboxColumnNames.UseVisualStyleBackColor = true;
+      this.CheckboxColumnNames.CheckedChanged += new System.EventHandler(this.CheckboxColumnNames_CheckedChanged);
       // 
       // DateTimeFormatInfoLabel
@@ -315 +366 @@
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TrainingTestTrackBar);
       this.ProblemDataSettingsGroupBox.Controls.Add(this.ShuffleDataCheckbox);
-      this.ProblemDataSettingsGroupBox.Location = new System.Drawing.Point(12, 164);
+      this.ProblemDataSettingsGroupBox.Location = new System.Drawing.Point(12, 191);
       this.ProblemDataSettingsGroupBox.Name = "ProblemDataSettingsGroupBox";
-      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 252);
+      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 225);
       this.ProblemDataSettingsGroupBox.TabIndex = 17;
       this.ProblemDataSettingsGroupBox.TabStop = false;
@@ -365 +416 @@
       this.PreviewDatasetMatrix.ShowRowsAndColumnsTextBox = false;
       this.PreviewDatasetMatrix.ShowStatisticalInformation = false;
-      this.PreviewDatasetMatrix.Size = new System.Drawing.Size(435, 138);
+      this.PreviewDatasetMatrix.Size = new System.Drawing.Size(435, 111);
       this.PreviewDatasetMatrix.TabIndex = 0;
-      // 
-      // CheckboxColumnNames
-      // 
-      this.CheckboxColumnNames.AutoSize = true;
-      this.CheckboxColumnNames.Location = new System.Drawing.Point(9, 103);
-      this.CheckboxColumnNames.Name = "CheckboxColumnNames";
-      this.CheckboxColumnNames.Size = new System.Drawing.Size(144, 17);
-      this.CheckboxColumnNames.TabIndex = 24;
-      this.CheckboxColumnNames.Text = "Column names in first line";
-      this.CheckboxColumnNames.UseVisualStyleBackColor = true;
-      this.CheckboxColumnNames.CheckedChanged += new System.EventHandler(this.CheckboxColumnNames_CheckedChanged);
       // 
       // DataAnalysisImportTypeDialog
@@ -437 +477 @@
     protected System.Windows.Forms.ToolTip ToolTip;
     private System.Windows.Forms.CheckBox CheckboxColumnNames;
+    protected System.Windows.Forms.Label EncodingInfoLabel;
+    protected System.Windows.Forms.Label EncodingLabel;
+    protected System.Windows.Forms.ComboBox EncodingComboBox;
   }
 }

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/DataAnalysisImportTypeDialog.cs

@@ -25 +25 @@
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Windows.Forms;
 using HeuristicLab.Core.Views;
@@ -32 +33 @@
   public partial class DataAnalysisImportTypeDialog : Form {
 
-    public static readonly List<KeyValuePair<DateTimeFormatInfo, string>> dateTimeFormats = new List<KeyValuePair<DateTimeFormatInfo, string>>{
-      new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE")), "dd/mm/yyyy hh:MM:ss" ),
-      new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "mm/dd/yyyy hh:MM:ss" ),
-      new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "yyyy/mm/dd hh:MM:ss" ),
-      new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "mm/yyyy/dd hh:MM:ss" )
+    private static readonly List<KeyValuePair<DateTimeFormatInfo, string>> dateTimeFormats =
+      new List<KeyValuePair<DateTimeFormatInfo, string>>{
+        new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.GetInstance(new CultureInfo("de-DE")), "dd/mm/yyyy hh:MM:ss" ),
+        new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "mm/dd/yyyy hh:MM:ss" ),
+        new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "yyyy/mm/dd hh:MM:ss" ),
+        new KeyValuePair<DateTimeFormatInfo, string>(DateTimeFormatInfo.InvariantInfo, "mm/yyyy/dd hh:MM:ss" )
     };
 
-    public static readonly List<KeyValuePair<char, string>> POSSIBLE_SEPARATORS = new List<KeyValuePair<char, string>>{  
-      new KeyValuePair<char, string>(';', "; (Semicolon)" ),
-      new KeyValuePair<char, string>(',', ", (Comma)" ),    
-      new KeyValuePair<char, string>('\t', "\\t (Tab)"),
-      new KeyValuePair<char, string>((char)0, "all whitespaces (including tabs and spaces)")
+    private static readonly List<KeyValuePair<char, string>> POSSIBLE_SEPARATORS =
+      new List<KeyValuePair<char, string>>{  
+        new KeyValuePair<char, string>(';', "; (Semicolon)" ),
+        new KeyValuePair<char, string>(',', ", (Comma)" ),    
+        new KeyValuePair<char, string>('\t', "\\t (Tab)"),
+        new KeyValuePair<char, string>((char)0, "all whitespaces (including tabs and spaces)")
     };
 
-    public static readonly List<KeyValuePair<NumberFormatInfo, string>> POSSIBLE_DECIMAL_SEPARATORS = new List<KeyValuePair<NumberFormatInfo, string>>{
-      new KeyValuePair<NumberFormatInfo, string>(NumberFormatInfo.GetInstance(new CultureInfo("de-DE")), ", (Comma)"),
-      new KeyValuePair<NumberFormatInfo, string>(NumberFormatInfo.InvariantInfo, ". (Period)" )    
+    private static readonly List<KeyValuePair<NumberFormatInfo, string>> POSSIBLE_DECIMAL_SEPARATORS =
+      new List<KeyValuePair<NumberFormatInfo, string>>{
+        new KeyValuePair<NumberFormatInfo, string>(NumberFormatInfo.GetInstance(new CultureInfo("de-DE")), ", (Comma)"),
+        new KeyValuePair<NumberFormatInfo, string>(NumberFormatInfo.InvariantInfo, ". (Period)" )    
     };
+
+    private static readonly List<KeyValuePair<Encoding, string>> POSSIBLE_ENCODINGS =
+      new List<KeyValuePair<Encoding, string>> {
+        new KeyValuePair<Encoding, string>(Encoding.Default, "Default"),
+        new KeyValuePair<Encoding, string>(Encoding.ASCII, "ASCII"),
+        new KeyValuePair<Encoding, string>(Encoding.Unicode, "Unicode"),    
+        new KeyValuePair<Encoding, string>(Encoding.UTF8, "UTF8")        
+      };
 
     public string Path {
@@ -87 +99 @@
       DateTimeFormatComboBox.ValueMember = "Key";
       DateTimeFormatComboBox.DisplayMember = "Value";
+      EncodingComboBox.DataSource = POSSIBLE_ENCODINGS;
+      EncodingComboBox.ValueMember = "Key";
+      EncodingComboBox.DisplayMember = "Value";
+
     }
 
@@ -100 +116 @@
       DecimalSeparatorComboBox.Enabled = true;
       DateTimeFormatComboBox.Enabled = true;
+      EncodingComboBox.Enabled = true;
       ProblemTextBox.Text = openFileDialog.FileName;
       TableFileParser csvParser = new TableFileParser();
@@ -125 +142 @@
       try {
         TableFileParser csvParser = new TableFileParser();
+        csvParser.Encoding = (Encoding)EncodingComboBox.SelectedValue;
         csvParser.Parse(ProblemTextBox.Text,
                         (NumberFormatInfo)DecimalSeparatorComboBox.SelectedValue,

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/RegressionImportTypeDialog.Designer.cs

@@ -66 +66 @@
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TargetVariableLabel);
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TargetVariableComboBox);
-      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 251);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.PreviewLabel, 0);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.ShuffelInfoLabel, 0);

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis.Views/3.3/TimeSeriesPrognosisImportTypeDialog.Designer.cs

@@ -58 +58 @@
       this.ShuffleDataCheckbox.Enabled = false;
       this.ShuffleDataCheckbox.Visible = false;
-      // 
-      // OkButton
-      // 
-      this.OkButton.Location = new System.Drawing.Point(303, 407);
-      // 
-      // CancelationButton
-      // 
-      this.CancelationButton.Location = new System.Drawing.Point(384, 407);
+
       // 
       // ProblemDataSettingsGroupBox
@@ -72 +65 @@
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TargetVariableComboBox);
       this.ProblemDataSettingsGroupBox.Controls.Add(this.TargetVariableLabel);
-      this.ProblemDataSettingsGroupBox.Size = new System.Drawing.Size(447, 237);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.ShuffleDataCheckbox, 0);
       this.ProblemDataSettingsGroupBox.Controls.SetChildIndex(this.TargetVariableLabel, 0);
@@ -89 +81 @@
       this.ErrorTextBox.Location = new System.Drawing.Point(6, 19);
       this.ErrorTextBox.Size = new System.Drawing.Size(435, 69);
-      // 
-      // PreviewDatasetMatrix
-      // 
-      this.PreviewDatasetMatrix.Size = new System.Drawing.Size(435, 123);
       // 
       // SeparatorInfoLabel
@@ -114 +102 @@
       // TargetVariableComboBox
       // 
-      this.TargetVariableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
+      this.TargetVariableComboBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
             | System.Windows.Forms.AnchorStyles.Right)));
       this.TargetVariableComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
@@ -148 +136 @@
       this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
       this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
-      this.ClientSize = new System.Drawing.Size(471, 442);
+      this.ClientSize = new System.Drawing.Size(471, 457);
       this.Name = "TimeSeriesPrognosisImportTypeDialog";
       this.Text = "TimeSeries Prognosis CSV Import";

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/DataAnalysisInstanceProvider.cs

@@ -24 +24 @@
 using System.Collections.Generic;
 using System.ComponentModel;
-using System.Drawing;
 using System.Globalization;
 using System.IO;
@@ -52 +51 @@
       var handler = ProgressChanged;
       if (handler != null)
-        handler(this, new ProgressChangedEventArgs((int)(100*d), null));
+        handler(this, new ProgressChangedEventArgs((int)(100 * d), null));
     }
 
@@ -102 +101 @@
         strBuilder.AppendLine();
       }
-
-      using (var writer = new StreamWriter(path)) {
-        writer.Write(strBuilder);
+      using (var fileStream = new FileStream(path, FileMode.Create)) {
+        using (var writer = new StreamWriter(fileStream, Encoding.Default)) {
+          writer.Write(strBuilder);
+        }
       }
     }

trunk/sources/HeuristicLab.Problems.Instances.DataAnalysis/3.3/TableFileParser.cs

@@ -28 +28 @@
 using System.IO;
 using System.Linq;
-using System.Runtime;
 using System.Runtime.Serialization;
 using System.Text;
@@ -41 +40 @@
     private int estimatedNumberOfLines = 200; // initial capacity for columns, will be set automatically when data is read from a file
 
+
+    private Encoding encoding = Encoding.Default;
+
+    public Encoding Encoding {
+      get { return encoding; }
+      set {
+        if (value == null) throw new ArgumentNullException("Encoding");
+        encoding = value;
+      }
+    }
+
+
     private int rows;
     public int Rows {
@@ -104 +115 @@
     public bool AreColumnNamesInFirstLine(Stream stream, NumberFormatInfo numberFormat,
                                           DateTimeFormatInfo dateTimeFormatInfo, char separator) {
-      using (StreamReader reader = new StreamReader(stream)) {
+      using (StreamReader reader = new StreamReader(stream, Encoding)) {
         tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
         return (tokenizer.PeekType() != TokenTypeEnum.Double);
@@ -143 +154 @@
       var len = new System.IO.FileInfo(fileName).Length;
       var buf = new char[1024 * 1024];
-      using (var reader = new StreamReader(fileName)) {
+      using (var reader = new StreamReader(fileName, Encoding)) {
         reader.ReadBlock(buf, 0, buf.Length);
       }
@@ -187 +198 @@
     /// <param name="columnNamesInFirstLine"></param>
     public void Parse(Stream stream, NumberFormatInfo numberFormat, DateTimeFormatInfo dateTimeFormatInfo, char separator, bool columnNamesInFirstLine, int lineLimit = -1) {
-      using (StreamReader reader = new StreamReader(stream)) {
+      using (StreamReader reader = new StreamReader(stream, Encoding)) {
         tokenizer = new Tokenizer(reader, numberFormat, dateTimeFormatInfo, separator);
         values = new List<IList>();
@@ -408 +419 @@
               .Where(c => OccurrencesOf(charCounts, c) > 10)
               .OrderBy(c => -OccurrencesOf(charCounts, c))
-              .DefaultIfEmpty(' ') 
+              .DefaultIfEmpty(' ')
               .First();
           }
@@ -505 +516 @@
           try {
             BytesRead = reader.BaseStream.Position;
-          } catch (IOException) {
+          }
+          catch (IOException) {
             BytesRead += CurrentLine.Length + 2; // guess
-          } catch (NotSupportedException) {
+          }
+          catch (NotSupportedException) {
             BytesRead += CurrentLine.Length + 2;
           }