Changeset 15843

Timestamp:

03/16/18 10:41:38 (7 years ago)

Author:

lkammere

Message:

#2886: Remove duplicates in logged sentences using bash commands.

Location:

branches/2886_SymRegGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration

Files:

• Analysis/SentenceLogger.cs (modified) (4 diffs)
• GrammarEnumeration/GrammarEnumerationAlgorithm.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration.csproj (modified) (1 diff)

branches/2886_SymRegGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration/Analysis/SentenceLogger.cs

@@ -7 +7 @@
 namespace HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration {
   class SentenceLogger : Item, IGrammarEnumerationAnalyzer {
-    private readonly string distinctSentencesFileName = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory) + @"\distinctSentences.csv";
-    private readonly string allSentencesFileName = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory) + @"\allSentences.csv";
+    private readonly string workingDir = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory);
     private readonly string columnDelimiter = ";";
-    private readonly string header = "infix;postfix;hash";
+    private readonly string header = "hash;length;postfix;infix";
+
+    private string distinctSentencesFileName;
+    private string allSentencesFileName;
+    private string shortestDistinctSentencesFileName;
 
     private TextWriterTraceListener distinctSentencesFileTrace;
@@ -42 +45 @@
 
     private void GrammarEnumerationAlgorithmOnStarted(object sender, EventArgs eventArgs) {
+      string datePostfix = $"_{DateTime.Now:yyyy-MM-dd_HH-mm}_TreeSize-{((GrammarEnumerationAlgorithm) sender).MaxTreeSize}.csv";
+      distinctSentencesFileName = workingDir + @"\distinctSentences" + datePostfix;
+      allSentencesFileName = workingDir + @"\allSentences" + datePostfix;
+      shortestDistinctSentencesFileName = workingDir + @"\shortestDistinctSentences" + datePostfix;
+
       distinctSentencesFileTrace = new TextWriterTraceListener(new FileStream(distinctSentencesFileName, FileMode.Create));
       allSentencesFileTrace = new TextWriterTraceListener(new FileStream(allSentencesFileName, FileMode.Create));
@@ -47 +55 @@
       ((StreamWriter)allSentencesFileTrace.Writer).AutoFlush = true;
 
-      distinctSentencesFileTrace.WriteLine(header);
       allSentencesFileTrace.WriteLine(header);
     }
@@ -54 +61 @@
       distinctSentencesFileTrace.Close();
       allSentencesFileTrace.Close();
+
+      // Remove duplicates afterwards using bash commands from the git bash
+      string bashExecutable = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + @"\Programs\Git\git-bash.exe";
+
+      string commandCreate = $"echo \"{header}\" > {shortestDistinctSentencesFileName.Replace("\\", "/")}";
+      string commandFill = $"sort -s {distinctSentencesFileName.Replace("\\", "/")} | uniq -w 10 >> {shortestDistinctSentencesFileName.Replace("\\", "/")}";
+      
+      ProcessStartInfo startInfo = new ProcessStartInfo {
+        WindowStyle = ProcessWindowStyle.Hidden,
+        UseShellExecute = false,
+        CreateNoWindow = true,
+        FileName = bashExecutable,
+        RedirectStandardError = true,
+        RedirectStandardOutput = true,
+        Arguments = $"-c '{commandCreate};{commandFill}'"
+      };
+      Process.Start(startInfo);
     }
 
     private void DistinctSentenceGenerated(object sender, PhraseAddedEventArgs phraseAddedEventArgs) {
       distinctSentencesFileTrace.WriteLine(ToCsvLine(
-        ((GrammarEnumerationAlgorithm)sender).Grammar.ToInfixString(phraseAddedEventArgs.NewPhrase),
+        ((uint)phraseAddedEventArgs.NewHash).ToString("D10"),
+        phraseAddedEventArgs.NewPhrase.Count().ToString("D3"),
         phraseAddedEventArgs.NewPhrase.ToString(),
-        phraseAddedEventArgs.NewHash.ToString()));
+        ((GrammarEnumerationAlgorithm)sender).Grammar.ToInfixString(phraseAddedEventArgs.NewPhrase)));
     }
 
     private void SentenceGenerated(object sender, PhraseAddedEventArgs phraseAddedEventArgs) {
       allSentencesFileTrace.WriteLine(ToCsvLine(
-        ((GrammarEnumerationAlgorithm)sender).Grammar.ToInfixString(phraseAddedEventArgs.NewPhrase),
+        ((uint)phraseAddedEventArgs.NewHash).ToString("D10"),
+        phraseAddedEventArgs.NewPhrase.Count().ToString("D3"),
         phraseAddedEventArgs.NewPhrase.ToString(),
-        phraseAddedEventArgs.NewHash.ToString()));
+        ((GrammarEnumerationAlgorithm)sender).Grammar.ToInfixString(phraseAddedEventArgs.NewPhrase)));
     }
 

branches/2886_SymRegGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration/GrammarEnumeration/GrammarEnumerationAlgorithm.cs

@@ -98 +98 @@
 
       Parameters.Add(new ValueParameter<IntValue>(MaxTreeSizeParameterName, "The number of clusters.", new IntValue(6)));
-      Parameters.Add(new ValueParameter<IntValue>(GuiUpdateIntervalParameterName, "Number of generated sentences, until GUI is refreshed.", new IntValue(1000)));
+      Parameters.Add(new ValueParameter<IntValue>(GuiUpdateIntervalParameterName, "Number of generated sentences, until GUI is refreshed.", new IntValue(100000)));
       Parameters.Add(new ValueParameter<EnumValue<StorageType>>(SearchDataStructureParameterName, new EnumValue<StorageType>(StorageType.Stack)));
 

branches/2886_SymRegGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration.csproj

@@ -17 +17 @@
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
-    <OutputPath>..\..\trunk\bin\</OutputPath>
+    <OutputPath>..\..\..\trunk\bin\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>