Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
08/10/09 15:51:04 (15 years ago)
Author:
gkronber
Message:

Worked on persistence of models into the DB. #719 (CEDMA Importer)

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/tools/CedmaImporter/Importer.cs

    r2265 r2272  
    55using System.IO;
    66using HeuristicLab.Modeling.Database;
    7 using HeuristicLab.Modeling.Database.SQLServerCompact;
    87using HeuristicLab.GP;
    98using HeuristicLab.GP.Interfaces;
     
    1918    private const int ALGORITHM_COLUMN = 3;
    2019    private const int RESULTS_IDX = 4;
    21     private Result[] results;
     20    private const int TRAINING_MSE = 4;
     21    private const int VALIDATION_MSE = TRAINING_MSE + 1;
     22    private const int TEST_MSE = TRAINING_MSE + 2;
     23
     24    private const int TRAINING_R2 = 7;
     25    private const int VALIDATION_R2 = TRAINING_R2 + 1;
     26    private const int TEST_R2 = TRAINING_R2 + 2;
     27
     28    private const int TRAINING_MAPE = 10;
     29    private const int VALIDATION_MAPE = TRAINING_MAPE + 1;
     30    private const int TEST_MAPE = TRAINING_MAPE + 2;
     31
     32    private const int TRAINING_MAPRE = 13;
     33    private const int VALIDATION_MAPRE = TRAINING_MAPRE + 1;
     34    private const int TEST_MAPRE = TRAINING_MAPRE + 2;
     35
     36    private const int TRAINING_VAF = 16;
     37    private const int VALIDATION_VAF = TRAINING_VAF + 1;
     38    private const int TEST_VAF = TRAINING_VAF + 2;
     39
     40    private const int VARIABLE_IMPACTS = 19;
     41    private const string EVALUATION_IMPACT = "EvaluationImpact";
     42    private const string QUALITY_IMPACT = "QualityImpact";
     43
     44    private string[] results;
    2245    private string[] inputVariables;
    23 
    2446    private HeuristicLab.CEDMA.Server.Problem problem;
    2547
     
    3052
    3153    public void Import(string fileName, string dirName) {
    32       string outputFileName = Path.GetFileNameWithoutExtension(fileName) + ".sdf";
     54      string outputFileName = Path.Combine(dirName, Path.GetFileNameWithoutExtension(fileName) + ".sdf");
    3355      string connectionString = @"Data Source=" + outputFileName;
    3456
    35       DatabaseService database = new DatabaseService(connectionString);
    36       Problem p = database.GetOrCreateProblem(problem.Dataset);
     57      var database = new HeuristicLab.Modeling.Database.SQLServerCompact.DatabaseService(connectionString);
     58      IProblem p = database.GetOrCreateProblem(problem.Dataset);
    3759      using (StreamReader reader = File.OpenText(fileName)) {
    38         ReadResultsAndInputVariables(reader);
     60        ReadResultsAndInputVariables(reader, database);
    3961        reader.ReadLine();
    4062        ImportAllModels(dirName, reader, database);
     
    4264    }
    4365
    44     private void ReadResultsAndInputVariables(StreamReader reader) {
    45       string[] columns = reader.ReadLine().Split(';');
    46       results = Enumerable.Repeat<Result>(null, columns.Length).ToArray();
    47       inputVariables = Enumerable.Repeat<string>(null, columns.Length).ToArray();
     66    private void ReadResultsAndInputVariables(StreamReader reader, IModelingDatabase database) {
     67      string[] columns = reader.ReadLine().Split(';').Select(x=>x.Trim()).ToArray();
     68      results = new string[columns.Length];
     69      inputVariables = new string[columns.Length];
    4870      for (int i = RESULTS_IDX; i < columns.Length; i++) {
    4971        string resultColumn = columns[i].Trim();
    50         if (resultColumn.Contains(":")) {
    51           string[] tokens = resultColumn.Split(':');
    52           string variableName = tokens[1].Trim();
     72        if (resultColumn.Contains(" ")) {
     73          string[] tokens = resultColumn.Split(' ');
     74          string variableName = tokens[1].Trim(' ','(',')');
    5375          string variableResultName = tokens[0].Trim();
    5476          inputVariables[i] = variableName;
    55           results[i] = new Result(variableResultName);
     77          results[i] = variableResultName;
    5678        } else {
    5779          // normal result value
    58           results[i] = new Result(resultColumn);
     80          results[i] = resultColumn;
    5981        }
    6082      }
    6183    }
    6284
    63     private void ImportAllModels(string dirName, StreamReader reader, DatabaseService database) {
     85    private void ImportAllModels(string dirName, StreamReader reader, IModelingDatabase database) {
    6486      while (!reader.EndOfStream) {
    65         string modelLine = reader.ReadLine();
    66         string[] modelData = modelLine.Split(';');
     87        string[] modelData = reader.ReadLine().Split(';').Select(x => x.Trim()).ToArray();
    6788        int id = int.Parse(modelData[ID_COLUMN]);
    6889        string targetVariableName = modelData[TARGETVARIABLE_COLUMN].Trim();
    6990        string algoName = modelData[ALGORITHM_COLUMN].Trim();
    7091        try {
    71           HeuristicLab.Core.IItem modelItem = ParseModel(dirName, modelData[FILENAME_COLUMN].Trim(), algoName);
    72           HeuristicLab.Modeling.Database.SQLServerCompact.Variable targetVariable = new HeuristicLab.Modeling.Database.SQLServerCompact.Variable(targetVariableName);
    73           Algorithm algorithm = new Algorithm(algoName);
    74           Model model = new Model(targetVariable, algorithm);
     92          HeuristicLab.Modeling.Model model = new HeuristicLab.Modeling.Model();
     93          model.TargetVariable = targetVariableName;
     94          model.Dataset = problem.Dataset;
    7595          model.TrainingSamplesStart = problem.TrainingSamplesStart;
    7696          model.TrainingSamplesEnd = problem.TrainingSamplesEnd;
     
    80100          model.TestSamplesEnd = problem.TestSamplesEnd;
    81101
    82           IEnumerable<ModelResult> qualityModelResults = GetModelResults(model, modelData);
    83           IEnumerable<InputVariableResult> inputVariableResults = GetInputVariableResults(model, modelData);
    84102
    85           // TODO
    86           //database.Persist(model);
    87           //foreach (ModelResult modelResult in qualityModelResults)
    88           //  database.Persist(modelResult);
    89           //foreach (InputVariableResult inputVariableResult in inputVariableResults)
    90           //  database.Persist(inputVariableResult);
     103          model.Data = ParseModel(dirName, modelData[FILENAME_COLUMN].Trim(), algoName);
     104
     105          SetModelResults(model, modelData);
     106          SetInputVariableResults(model, modelData);
     107
     108          database.Persist(model, algoName, null);
    91109        }
    92110        catch (Exception ex) {
     
    95113    }
    96114
    97     private IEnumerable<InputVariableResult> GetInputVariableResults(Model model, string[] modelData) {
    98       double temp;
    99       return from i in Enumerable.Range(0, inputVariables.Count())
    100              where inputVariables[i] != null && results[i] != null && double.TryParse(modelData[i], out temp)
    101              select new InputVariableResult(new InputVariable(model, new HeuristicLab.Modeling.Database.SQLServerCompact.Variable(inputVariables[i])), results[i], double.Parse(modelData[i]));
     115    private void SetInputVariableResults(HeuristicLab.Modeling.Model model, string[] modelData) {
     116      for (int i = VARIABLE_IMPACTS; i < modelData.Length; i++) {
     117        if (!string.IsNullOrEmpty(modelData[i])) {
     118          model.AddInputVariables(inputVariables[i]);
     119          if (results[i] == EVALUATION_IMPACT) {
     120            model.SetVariableEvaluationImpact(inputVariables[i], double.Parse(modelData[i]));
     121          } else if (results[i] == QUALITY_IMPACT) {
     122            model.SetVariableQualityImpact(inputVariables[i], double.Parse(modelData[i]));
     123          } else throw new FormatException();
     124        }
     125      }
    102126    }
    103127
    104     private IEnumerable<ModelResult> GetModelResults(Model model, string[] modelData) {
    105       return from i in Enumerable.Range(0, results.Count())
    106              where results[i] != null
    107              select new ModelResult(model, results[i], double.Parse(modelData[i]));
     128    private void SetModelResults(HeuristicLab.Modeling.Model model, string[] modelData) {
     129      model.TrainingMeanSquaredError = double.Parse(modelData[TRAINING_MSE]);
     130      model.ValidationMeanSquaredError = double.Parse(modelData[VALIDATION_MSE]);
     131      model.TestMeanSquaredError = double.Parse(modelData[TEST_MSE]);
     132
     133      model.TrainingCoefficientOfDetermination = double.Parse(modelData[TRAINING_R2]);
     134      model.ValidationCoefficientOfDetermination = double.Parse(modelData[VALIDATION_R2]);
     135      model.TestCoefficientOfDetermination = double.Parse(modelData[TEST_R2]);
     136
     137      model.TrainingMeanAbsolutePercentageError = double.Parse(modelData[TRAINING_MAPE]);
     138      model.ValidationCoefficientOfDetermination = double.Parse(modelData[VALIDATION_MAPE]);
     139      model.TestCoefficientOfDetermination = double.Parse(modelData[TEST_MAPE]);
     140
     141      model.TrainingMeanAbsolutePercentageOfRangeError = double.Parse(modelData[TRAINING_MAPRE]);
     142      model.ValidationMeanAbsolutePercentageOfRangeError = double.Parse(modelData[VALIDATION_MAPRE]);
     143      model.TestMeanAbsolutePercentageOfRangeError = double.Parse(modelData[TEST_MAPRE]);
     144
     145      model.TrainingVarianceAccountedFor = double.Parse(modelData[TRAINING_VAF]);
     146      model.ValidationVarianceAccountedFor = double.Parse(modelData[VALIDATION_VAF]);
     147      model.TestVarianceAccountedFor = double.Parse(modelData[TEST_VAF]);
    108148    }
    109149
Note: See TracChangeset for help on using the changeset viewer.