Free cookie consent management tool by TermsFeed Policy Generator

Changeset 1216


Ignore:
Timestamp:
02/12/09 14:05:33 (15 years ago)
Author:
gkronber
Message:

Improved dispatching mechanism. #419 (Refactor CEDMA plugins).

Location:
branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server/Dispatcher.cs

    r1151 r1216  
    5757      };
    5858
    59       var dataSetBindings = store.Query("?DataSet <"+Ontology.PredicateInstanceOf.Uri+"> <"+Ontology.TypeDataSet.Uri+"> .");
     59      var dataSetBindings = store.Query("?DataSet <" + Ontology.PredicateInstanceOf.Uri + "> <" + Ontology.TypeDataSet.Uri + "> .");
    6060
    6161      // no datasets => do nothing
    … …  
    6565      // find and select all results for this dataset
    6666      var dataSetEntity = (Entity)dataSetBindings.Last().Get("DataSet");
    67       var targetVar = new HeuristicLab.CEDMA.DB.Interfaces.Variable("TargetVariable");
    68       var modelVar = new HeuristicLab.CEDMA.DB.Interfaces.Variable("Model");
    69       var modelMAPE = new HeuristicLab.CEDMA.DB.Interfaces.Variable("ModelMAPE");
    70 
    71       var query = "<" + dataSetEntity.Uri + "> <" + Ontology.PredicateHasModel.Uri + "> ?Model ." + Environment.NewLine +
    72         "?Model <" + Ontology.TargetVariable.Uri + "> ?TargetVariable ." + Environment.NewLine +
    73         "?Model <" + Ontology.ValidationMeanAbsolutePercentageError.Uri + "> ?ModelMAPE .";
    74 
    75 
    76 
    77       var bindings = store.Query(query);
    7867      DataSet dataSet = new DataSet(store, dataSetEntity);
    79       double[] utilization = new double[dataSet.Problem.AllowedTargetVariables.Count];
    80       int i = 0;
    81       int totalN = bindings.Count();
    82       foreach (int targetVariable in dataSet.Problem.AllowedTargetVariables) {
    83         var targetVarBindings = bindings.Where(x => (int)((Literal)x.Get("TargetVariable")).Value == targetVariable);
    84         if (targetVarBindings.Count() == 0) {
    85           utilization[i++] = double.PositiveInfinity;
    86         } else {
    87           double averageMape = targetVarBindings.Average(x => (double)((Literal)x.Get("ModelMAPE")).Value);
    88           double n = targetVarBindings.Count();
    89           utilization[i++] = -averageMape + Math.Sqrt(Math.Log(totalN) / n) * 0.1;
    90         }
    91       }
    92       int[] idx = Enumerable.Range(0, utilization.Length).ToArray();
    93       Array.Sort(utilization, idx);
    94       int nConfigurations = utilization.Length;
    95       for (int j = nConfigurations - 1; j > nConfigurations * 0.8; j--) {
    96         int targetVariable = dataSet.Problem.AllowedTargetVariables[idx[j]];
    97         IEngine engine = CreateEngine(dataSet.Problem, targetVariable);
    98         if (engine != null) {
    99           QueueJob(new Execution(dataSetEntity, engine, targetVariable));
    100         }
     68      Random random = new Random();
     69      int targetVariable = dataSet.Problem.AllowedInputVariables[random.Next(0, dataSet.Problem.AllowedInputVariables.Count)];
     70      string targetVariableName = dataSet.Problem.GetVariableName(targetVariable);
     71      Execution exec = CreateExecution(dataSet.Problem, targetVariable);
     72      if (exec != null) {
     73        exec.DataSetEntity = dataSetEntity;
     74        exec.TargetVariable = targetVariableName;
     75        QueueJob(exec);
    10176      }
    10277    }
    … …  
    12297    }
    12398
    124     private IEngine CreateEngine(Problem problem, int targetVariable) {
     99    private Execution CreateExecution(Problem problem, int targetVariable) {
    125100      switch (problem.LearningTask) {
    126101        case LearningTask.Classification: return null;
    127102        case LearningTask.Regression: {
    128             return CreateStandardGp(problem, targetVariable).Engine;
     103            return CreateStandardGpExecution(problem, targetVariable, 100, 10);
    129104          }
    130105        case LearningTask.TimeSeries: return null;
    … …  
    134109    }
    135110
    136     private StandardGP CreateStandardGp(Problem problem, int targetVariable) {
     111    private Execution CreateStandardGpExecution(Problem problem, int targetVariable, int maxTreeSize, int maxTreeHeight) {
    137112      ProblemInjector probInjector = new ProblemInjector(problem);
    138113      probInjector.TargetVariable = targetVariable;
    139114      StandardGP sgp = new StandardGP();
    140115      sgp.SetSeedRandomly = true;
    141       sgp.MaxGenerations = 300;
    142       sgp.PopulationSize = 10000;
     116      sgp.MaxGenerations = 2;
     117      sgp.PopulationSize = 100;
    143118      sgp.Elites = 1;
    144119      sgp.ProblemInjector = probInjector;
    145       return sgp;
     120      sgp.MaxTreeHeight = maxTreeHeight;
     121      sgp.MaxTreeSize = maxTreeSize;
     122      Execution exec = new Execution(sgp.Engine);
     123      exec.Description = "StandardGP - Medium Complexity";
     124      return exec;
    146125    }
    147126  }

  • branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server/Executer.cs

    r1156 r1216  
    4141namespace HeuristicLab.CEDMA.Server {
    4242  public class Executer {
    43     private static int MaxActiveJobs {
    44       get { return 100; }
    45     }
    4643    private Dispatcher dispatcher;
    4744    private JobManager jobManager;
    4845    private IStore store;
     46    private Dictionary<WaitHandle, Execution> activeExecutions;
     47
     48    private TimeSpan StartJobInterval {
     49      get { return TimeSpan.FromMilliseconds(500); }
     50    }
     51
     52    private TimeSpan WaitForFinishedJobsTimeout {
     53      get { return TimeSpan.FromMilliseconds(100); }
     54    }
     55
     56    private int maxActiveJobs;
     57    public int MaxActiveJobs {
     58      get { return maxActiveJobs; }
     59      set {
     60        if (value < 0) throw new ArgumentException("Only positive values are allowed for MaxActiveJobs");
     61        maxActiveJobs = value;
     62      }
     63    }
    4964
    5065    public Executer(Dispatcher dispatcher, IStore store, string gridUrl) {
     66      activeExecutions = new Dictionary<WaitHandle, Execution>();
     67      maxActiveJobs = 10;
    5168      this.dispatcher = dispatcher;
    5269      this.store = store;
    … …  
    6178    private void StartJobs() {
    6279      List<WaitHandle> wh = new List<WaitHandle>();
    63       Dictionary<WaitHandle, AtomicOperation> activeOperations = new Dictionary<WaitHandle,AtomicOperation>();
    64       Dictionary<WaitHandle, Execution> activeExecutions = new Dictionary<WaitHandle,Execution>();
     80      Dictionary<WaitHandle, AtomicOperation> activeOperations = new Dictionary<WaitHandle, AtomicOperation>();
    6581      while (true) {
    6682        try {
    6783          // start new jobs as long as there are less than MaxActiveJobs
    6884          while (wh.Count < MaxActiveJobs) {
     85            Thread.Sleep(StartJobInterval);
    6986            // get an execution from the dispatcher and execute in grid via job-manager
    7087            Execution execution = dispatcher.GetNextJob();
    … …  
    7491              wh.Add(opWh);
    7592              activeOperations.Add(opWh, op);
    76               activeExecutions.Add(opWh, execution);
     93              lock (activeExecutions) {
     94                activeExecutions.Add(opWh, execution);
     95              }
    7796            }
    7897          }
    7998          // wait until any job is finished
    8099          WaitHandle[] whArr = wh.ToArray();
    81           int readyHandleIndex = WaitHandle.WaitAny(whArr);
    82           WaitHandle readyHandle = whArr[readyHandleIndex];
    83           AtomicOperation finishedOp = activeOperations[readyHandle];
    84           Execution finishedExecution = activeExecutions[readyHandle];
    85           wh.Remove(readyHandle);
    86           activeExecutions.Remove(readyHandle);
    87           activeOperations.Remove(readyHandle);
    88           ProcessingEngine finishedEngine = null;
    89           try {
    90             finishedEngine = jobManager.EndExecuteOperation(finishedOp);
    91           } catch (Exception badEx) {
    92             Trace.WriteLine("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
    93           }
    94           if (finishedEngine != null) {
    95             StoreResults(finishedExecution, finishedEngine);
     100          int readyHandleIndex = WaitHandle.WaitAny(whArr, WaitForFinishedJobsTimeout);
     101          if (readyHandleIndex != WaitHandle.WaitTimeout) {
     102            WaitHandle readyHandle = whArr[readyHandleIndex];
     103            AtomicOperation finishedOp = activeOperations[readyHandle];
     104            wh.Remove(readyHandle);
     105            Execution finishedExecution = null;
     106            lock (activeExecutions) {
     107              finishedExecution = activeExecutions[readyHandle];
     108              activeExecutions.Remove(readyHandle);
     109            }
     110            activeOperations.Remove(readyHandle);
     111            ProcessingEngine finishedEngine = null;
     112            try {
     113              finishedEngine = jobManager.EndExecuteOperation(finishedOp);
     114            }
     115            catch (Exception badEx) {
     116              Trace.WriteLine("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
     117            }
     118            if (finishedEngine != null) {
     119              StoreResults(finishedExecution, finishedEngine);
     120            }
    96121          }
    97122        }
    … …  
    115140      StoreModelAttribute(model, Ontology.TreeHeight, bestModelScope.GetVariableValue<IntData>("TreeHeight", false).Data);
    116141      StoreModelAttribute(model, Ontology.EvaluatedSolutions, bestModelScope.GetVariableValue<IntData>("EvaluatedSolutions", false).Data);
    117        
     142
    118143      byte[] serializedModel = PersistenceManager.SaveToGZip(bestModelScope.GetVariableValue("FunctionTree", false));
    119144      store.Add(new Statement(model, Ontology.PredicateSerializedData, new Literal(Convert.ToBase64String(serializedModel))));
    120    }
     145    }
    121146
    122147    private void StoreModelAttribute(Entity model, Entity predicate, object value) {
    123148      store.Add(new Statement(model, predicate, new Literal(value)));
    124149    }
     150
     151    internal string[] GetJobs() {
     152      lock (activeExecutions) {
     153        string[] retVal = new string[activeExecutions.Count];
     154        int i = 0;
     155        foreach (Execution e in activeExecutions.Values) {
     156          retVal[i++] = "Target-Variable: "+e.TargetVariable+" "+e.Description;
     157        }
     158        return retVal;
     159      }
     160    }
    125161  }
    126162}

  • branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server/Execution.cs

    r1214 r1216  
    5959    }
    6060
    61     public Execution(Entity dataSetEntity, IEngine engine, string targetVariable) {
    62       this.dataSetEntity = dataSetEntity;
     61    private string description;
     62    public string Description {
     63      get { return description; }
     64      set { description = value; }
     65    }
     66
     67    public Execution(IEngine engine) {
    6368      this.engine = engine;
    64       this.targetVariable = targetVariable;
    6569    }
    6670  }

  • branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server/ServerForm.cs

    r1060 r1216  
    4949    private static readonly string rdfFile = AppDomain.CurrentDomain.BaseDirectory + "rdf_store.db3";
    5050    private static readonly string rdfConnectionString = "sqlite:rdf:Data Source=\"" + rdfFile + "\"";
    51    
     51
    5252    public ServerForm() {
    5353      InitializeComponent();
    … …  
    5656      server.Start();
    5757      addressTextBox.Text = server.CedmaServiceUrl;
    58       dispatcher = new Dispatcher(store);
    59       dispatcher.Start();
    6058    }
    6159
    6260    private void refreshTimer_Tick(object sender, EventArgs e) {
    63       listBox.DataSource = dispatcher.DispatchQueue;
     61      listBox.DataSource = executer.GetJobs();
    6462    }
    6563
    6664    private void connectButton_Click(object sender, EventArgs e) {
     65      dispatcher = new Dispatcher(store);
     66      dispatcher.Start();
    6767      executer = new Executer(dispatcher, store, gridAddress.Text);
    6868      executer.Start();
     69      maxActiveJobsUpDown.Enabled = true;
     70      maxActiveJobsUpDown.Value = executer.MaxActiveJobs;
    6971      connectButton.Enabled = false;
     72      refreshTimer.Start();
     73    }
     74
     75    private void maxActiveJobsUpDown_ValueChanged(object sender, EventArgs e) {
     76      executer.MaxActiveJobs = Convert.ToInt32(maxActiveJobsUpDown.Value);
    7077    }
    7178  }

  • branches/CEDMA-Refactoring-Ticket419/HeuristicLab.CEDMA.Server/ServerForm.designer.cs

    r1044 r1216  
    5353      this.listBox = new System.Windows.Forms.ListBox();
    5454      this.refreshTimer = new System.Windows.Forms.Timer(this.components);
     55      this.maxActiveJobsUpDown = new System.Windows.Forms.NumericUpDown();
     56      this.label1 = new System.Windows.Forms.Label();
     57      ((System.ComponentModel.ISupportInitialize)(this.maxActiveJobsUpDown)).BeginInit();
    5558      this.SuspendLayout();
    5659      //
    … …  
    104107                  | System.Windows.Forms.AnchorStyles.Right)));
    105108      this.listBox.FormattingEnabled = true;
    106       this.listBox.Location = new System.Drawing.Point(12, 58);
     109      this.listBox.Location = new System.Drawing.Point(12, 84);
    107110      this.listBox.Name = "listBox";
    108       this.listBox.Size = new System.Drawing.Size(350, 277);
     111      this.listBox.Size = new System.Drawing.Size(350, 251);
    109112      this.listBox.TabIndex = 11;
    110113      //
    111114      // refreshTimer
    112115      //
    113       this.refreshTimer.Enabled = true;
    114116      this.refreshTimer.Interval = 1000;
    115117      this.refreshTimer.Tick += new System.EventHandler(this.refreshTimer_Tick);
     118      //
     119      // maxActiveJobsUpDown
     120      //
     121      this.maxActiveJobsUpDown.Enabled = false;
     122      this.maxActiveJobsUpDown.Location = new System.Drawing.Point(106, 59);
     123      this.maxActiveJobsUpDown.Maximum = new decimal(new int[] {
     124            64,
     125            0,
     126            0,
     127            0});
     128      this.maxActiveJobsUpDown.Name = "maxActiveJobsUpDown";
     129      this.maxActiveJobsUpDown.Size = new System.Drawing.Size(120, 20);
     130      this.maxActiveJobsUpDown.TabIndex = 12;
     131      this.maxActiveJobsUpDown.ValueChanged += new System.EventHandler(this.maxActiveJobsUpDown_ValueChanged);
     132      //
     133      // label1
     134      //
     135      this.label1.AutoSize = true;
     136      this.label1.Enabled = false;
     137      this.label1.Location = new System.Drawing.Point(12, 61);
     138      this.label1.Name = "label1";
     139      this.label1.Size = new System.Drawing.Size(84, 13);
     140      this.label1.TabIndex = 13;
     141      this.label1.Text = "&Max active jobs:";
    116142      //
    117143      // ServerForm
    … …  
    120146      this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
    121147      this.ClientSize = new System.Drawing.Size(374, 342);
     148      this.Controls.Add(this.label1);
     149      this.Controls.Add(this.maxActiveJobsUpDown);
    122150      this.Controls.Add(this.listBox);
    123151      this.Controls.Add(this.connectButton);
    … …  
    128156      this.Name = "ServerForm";
    129157      this.Text = "CEDMA Server";
     158      ((System.ComponentModel.ISupportInitialize)(this.maxActiveJobsUpDown)).EndInit();
    130159      this.ResumeLayout(false);
    131160      this.PerformLayout();
    … …  
    142171    private System.Windows.Forms.ListBox listBox;
    143172    private System.Windows.Forms.Timer refreshTimer;
     173    private System.Windows.Forms.NumericUpDown maxActiveJobsUpDown;
     174    private System.Windows.Forms.Label label1;
    144175  }
    145176}
Note: See TracChangeset for help on using the changeset viewer.