source: trunk/tools/CedmaImporter/Importer.cs @ 2265

Last change on this file since 2265 was 2265, checked in by gkronber, 12 years ago

Worked on CEDMA importer. #719

File size: 5.7 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using System.Text;
5using System.IO;
6using HeuristicLab.Modeling.Database;
7using HeuristicLab.Modeling.Database.SQLServerCompact;
8using HeuristicLab.GP;
9using HeuristicLab.GP.Interfaces;
10using HeuristicLab.GP.StructureIdentification;
11using System.Diagnostics;
12
13namespace CedmaImporter {
14  public class Importer {
15
16    private const int ID_COLUMN = 0;
17    private const int FILENAME_COLUMN = 1;
18    private const int TARGETVARIABLE_COLUMN = 2;
19    private const int ALGORITHM_COLUMN = 3;
20    private const int RESULTS_IDX = 4;
21    private Result[] results;
22    private string[] inputVariables;
23
24    private HeuristicLab.CEDMA.Server.Problem problem;
25
26
27    public Importer(HeuristicLab.CEDMA.Server.Problem problem) {
28      this.problem = problem;
29    }
30
31    public void Import(string fileName, string dirName) {
32      string outputFileName = Path.GetFileNameWithoutExtension(fileName) + ".sdf";
33      string connectionString = @"Data Source=" + outputFileName;
34
35      DatabaseService database = new DatabaseService(connectionString);
36      Problem p = database.GetOrCreateProblem(problem.Dataset);
37      using (StreamReader reader = File.OpenText(fileName)) {
38        ReadResultsAndInputVariables(reader);
39        reader.ReadLine();
40        ImportAllModels(dirName, reader, database);
41      }
42    }
43
44    private void ReadResultsAndInputVariables(StreamReader reader) {
45      string[] columns = reader.ReadLine().Split(';');
46      results = Enumerable.Repeat<Result>(null, columns.Length).ToArray();
47      inputVariables = Enumerable.Repeat<string>(null, columns.Length).ToArray();
48      for (int i = RESULTS_IDX; i < columns.Length; i++) {
49        string resultColumn = columns[i].Trim();
50        if (resultColumn.Contains(":")) {
51          string[] tokens = resultColumn.Split(':');
52          string variableName = tokens[1].Trim();
53          string variableResultName = tokens[0].Trim();
54          inputVariables[i] = variableName;
55          results[i] = new Result(variableResultName);
56        } else {
57          // normal result value
58          results[i] = new Result(resultColumn);
59        }
60      }
61    }
62
63    private void ImportAllModels(string dirName, StreamReader reader, DatabaseService database) {
64      while (!reader.EndOfStream) {
65        string modelLine = reader.ReadLine();
66        string[] modelData = modelLine.Split(';');
67        int id = int.Parse(modelData[ID_COLUMN]);
68        string targetVariableName = modelData[TARGETVARIABLE_COLUMN].Trim();
69        string algoName = modelData[ALGORITHM_COLUMN].Trim();
70        try {
71          HeuristicLab.Core.IItem modelItem = ParseModel(dirName, modelData[FILENAME_COLUMN].Trim(), algoName);
72          HeuristicLab.Modeling.Database.SQLServerCompact.Variable targetVariable = new HeuristicLab.Modeling.Database.SQLServerCompact.Variable(targetVariableName);
73          Algorithm algorithm = new Algorithm(algoName);
74          Model model = new Model(targetVariable, algorithm);
75          model.TrainingSamplesStart = problem.TrainingSamplesStart;
76          model.TrainingSamplesEnd = problem.TrainingSamplesEnd;
77          model.ValidationSamplesStart = problem.ValidationSamplesStart;
78          model.ValidationSamplesEnd = problem.ValidationSamplesEnd;
79          model.TestSamplesStart = problem.TestSamplesStart;
80          model.TestSamplesEnd = problem.TestSamplesEnd;
81
82          IEnumerable<ModelResult> qualityModelResults = GetModelResults(model, modelData);
83          IEnumerable<InputVariableResult> inputVariableResults = GetInputVariableResults(model, modelData);
84
85          // TODO
86          //database.Persist(model);
87          //foreach (ModelResult modelResult in qualityModelResults)
88          //  database.Persist(modelResult);
89          //foreach (InputVariableResult inputVariableResult in inputVariableResults)
90          //  database.Persist(inputVariableResult);
91        }
92        catch (Exception ex) {
93        }
94      }
95    }
96
97    private IEnumerable<InputVariableResult> GetInputVariableResults(Model model, string[] modelData) {
98      double temp;
99      return from i in Enumerable.Range(0, inputVariables.Count())
100             where inputVariables[i] != null && results[i] != null && double.TryParse(modelData[i], out temp)
101             select new InputVariableResult(new InputVariable(model, new HeuristicLab.Modeling.Database.SQLServerCompact.Variable(inputVariables[i])), results[i], double.Parse(modelData[i]));
102    }
103
104    private IEnumerable<ModelResult> GetModelResults(Model model, string[] modelData) {
105      return from i in Enumerable.Range(0, results.Count())
106             where results[i] != null
107             select new ModelResult(model, results[i], double.Parse(modelData[i]));
108    }
109
110    private HeuristicLab.Core.IItem ParseModel(string dirName, string modelFileName, string algoName) {
111      foreach (char c in Path.GetInvalidFileNameChars()) {
112        modelFileName = modelFileName.Replace(c, '_');
113      }
114      if (algoName == "SupportVectorRegression") {
115        HeuristicLab.Data.SVMModel model = new HeuristicLab.Data.SVMModel();
116        model.Model = SVM.Model.Read(Path.Combine(dirName, modelFileName) + ".svm.model.txt");
117        model.RangeTransform = SVM.RangeTransform.Read(Path.Combine(dirName, modelFileName) + ".svm.transform.txt");
118        return model;
119      } else {
120        SymbolicExpressionImporter sexpImporter = new SymbolicExpressionImporter();
121        GeneticProgrammingModel model = new GeneticProgrammingModel();
122        using (StreamReader reader = File.OpenText(Path.Combine(dirName, modelFileName) + ".gp.txt")) {
123          model.FunctionTree = sexpImporter.Import(reader);
124        }
125        return model;
126      }
127    }
128  }
129}
Note: See TracBrowser for help on using the repository browser.