Changeset 406

Timestamp:

07/29/08 19:17:36 (16 years ago)

Author:

gkronber

Message:

fixed #209 by changing the import format while keeping compatibility with HL2 exported files

File:

• trunk/sources/HeuristicLab.DataAnalysis/DatasetParser.cs (modified) (7 diffs)

trunk/sources/HeuristicLab.DataAnalysis/DatasetParser.cs

@@ -194 +194 @@
         using(StreamReader reader = new StreamReader(importFileName)) {
           tokenizer = new Tokenizer(reader, numberFormat);
-          tokenizer.Separators = new string[] { " ", ";", "\t" };
           try {
             // parse the file
@@ -210 +209 @@
     #region tokenizer
     internal enum TokenTypeEnum {
-      At, Assign, NewLine, String, Double, Int
+      At, Assign, NewLine, String, Double, Int, WhiteSpace
     }
 
@@ -235 +234 @@
       private StreamReader reader;
       private List<Token> tokens;
-      private string[] separators;
+      private string[] separators = new string[] { "@", "=", ";", "\t" };
       private NumberFormatInfo numberFormatInfo;
 
@@ -244 +243 @@
       public static Token AtToken = new Token(TokenTypeEnum.At, "@");
       public static Token AssignmentToken = new Token(TokenTypeEnum.Assign, "=");
-
+      public static Token SeparatorToken = new Token(TokenTypeEnum.WhiteSpace, "");
       public string[] Separators {
         get { return separators; }
@@ -262 +261 @@
           CurrentLine = reader.ReadLine();
           Token[] newTokens = Array.ConvertAll(CurrentLine.Split(separators, StringSplitOptions.RemoveEmptyEntries), delegate(string str) {
-            return MakeToken(str);
+            return MakeToken(str.Trim());
           });
 
-          tokens.AddRange(newTokens);
+          foreach(Token tok in newTokens) {
+            if(tok != SeparatorToken) tokens.Add(tok);
+          }
           tokens.Add(NewlineToken);
           CurrentLineNumber++;
@@ -272 +273 @@
 
       private Token MakeToken(string strToken) {
-        if(strToken == "@")
-          return AtToken;
-        else if(strToken == "=")
-          return AssignmentToken;
-        else {
-          Token token = new Token(TokenTypeEnum.String, strToken);
-
-          if(int.TryParse(strToken, NumberStyles.Integer, numberFormatInfo, out token.intValue)) {
-            token.type = TokenTypeEnum.Int;
-            return token;
-          } else if(double.TryParse(strToken, NumberStyles.Float, numberFormatInfo, out token.doubleValue)) {
-            token.type = TokenTypeEnum.Double;
-            return token;
-          }
-          // couldn't parse the token as an int or float number
+        Token token = new Token(TokenTypeEnum.String, strToken);
+
+        // try to parse as a number first
+        if(int.TryParse(strToken, NumberStyles.Integer, numberFormatInfo, out token.intValue)) {
+          token.type = TokenTypeEnum.Int;
           return token;
-        }
+        } else if(double.TryParse(strToken, NumberStyles.Float, numberFormatInfo, out token.doubleValue)) {
+          token.type = TokenTypeEnum.Double;
+          return token;
+        }
+        // couldn't parse the token as an int or float number so return a string token
+        return token;
       }
 
@@ -363 +359 @@
 
     private void ParseMetaData(bool strict) {
-      while(tokenizer.Peek() == Tokenizer.AtToken) {
-        Expect(Tokenizer.AtToken);
-
+      while(tokenizer.Peek().type==TokenTypeEnum.String) {
         Token nameToken = tokenizer.Next();
         if(nameToken.type != TokenTypeEnum.String)
           Error("Expected a variable name.", nameToken.stringValue, tokenizer.CurrentLineNumber);
-
-        Expect(Tokenizer.AssignmentToken);
 
         List<Token> tokens = new List<Token>();