Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
08/02/15 01:14:55 (9 years ago)
Author:
abeham
Message:

#2431:

  • removed output of RTs, RTus, ...
  • added 2nd row to charts that state restarts until a target was achieved
  • added option to include or exclude solution results
File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/PerformanceComparison/HeuristicLab.Analysis/3.3/Optimizers/IRRestarter.cs

    r12813 r12825  
    4444    private const string BestQualityResultName = "BestQuality";
    4545    private const string RandomRestartsResultName = "RandomRestarts";
    46     private const string RTsResultName = "RTs";
    47     private const string RTusResultName = "RTus";
    48     private const string FEsResultName = "FEs";
    49     private const string FEusResultName = "FEus";
     46    private const string EvaluatedSolutionsResultName = "EvaluatedSolutions";
     47    private const string EvaluatedMovesResultName = "EvaluatedMoves";
     48    private const string QualityPerClockResultName = "QualityPerClock";
     49    private const string QualityPerEvaluationsResultName = "QualityPerEvaluations";
    5050
    5151    public string Filename { get; set; }
    … …  
    124124        perEvaluationsAnalyzer.MoveCostPerSolutionParameter.Value = new DoubleValue(moveCostPerSolution);
    125125        OnPropertyChanged("MoveCostPerSolution");
     126      }
     127    }
     128
     129    [Storable]
     130    private bool storeSolutionInRun;
     131    public bool StoreSolutionInRun {
     132      get { return storeSolutionInRun; }
     133      set {
     134        if (storeSolutionInRun == value) return;
     135        storeSolutionInRun = value;
     136        OnPropertyChanged("StoreSolutionInRun");
    126137      }
    127138    }
    … …  
    196207      get { return algorithm; }
    197208      set {
    198         if (value != null && !value.ProblemType.IsAssignableFrom(typeof(ISingleObjectiveHeuristicOptimizationProblem)))
    199           throw new ArgumentException("Algorithm is not single-objective!");
    200209        if (algorithm == value) return;
    201210        if (algorithm != null) {
    … …  
    266275      maximumEvaluations = original.maximumEvaluations;
    267276      moveCostPerSolution = original.moveCostPerSolution;
     277      storeSolutionInRun = original.storeSolutionInRun;
    268278      targetValue = original.targetValue;
    269279      maximization = original.maximization;
    … …  
    291301      maximumEvaluations = 10000000; // 10 mio
    292302      moveCostPerSolution = 1;
     303      storeSolutionInRun = false;
    293304      targetValue = 0;
    294305      maximization = false;
    … …  
    311322      maximumEvaluations = 10000000; // 10 mio
    312323      moveCostPerSolution = 1;
     324      storeSolutionInRun = false;
    313325      targetValue = 0;
    314326      maximization = false;
    … …  
    330342      maximumEvaluations = 10000000; // 10 mio
    331343      moveCostPerSolution = 1;
     344      storeSolutionInRun = false;
    332345      targetValue = 0;
    333346      maximization = false;
    … …  
    391404        if (!CurrentRun.Results.ContainsKey(ExecutionTimeResultName))
    392405          CurrentRun.Results.Add(ExecutionTimeResultName, new TimeSpanValue(TimeSpan.Zero));
    393         CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, new IntValue(0));
    394         CurrentRun.Results.Add(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, new IntValue(0));
     406        // use double instead of int, otherwise one might run into int.MaxValue (at least with moves)
     407        CurrentRun.Results.Add(EvaluatedSolutionsResultName, new DoubleValue(0));
     408        CurrentRun.Results.Add(EvaluatedMovesResultName, new DoubleValue(0));
    395409        CurrentRun.Results.Add(BestQualityResultName, new DoubleValue(Maximization ? double.MinValue : double.MaxValue));
    396410        CurrentRun.Results.Add(RandomRestartsResultName, new IntValue(0));
    397         CurrentRun.Results.Add(RTsResultName, new DoubleValue(0));
    398         CurrentRun.Results.Add(RTusResultName, new DoubleValue(0));
    399         CurrentRun.Results.Add(FEsResultName, new DoubleValue(0));
    400         CurrentRun.Results.Add(FEusResultName, new DoubleValue(0));
    401       }
     411      }
     412      if (Algorithm.ExecutionState == ExecutionState.Stopped)
     413        Algorithm.Prepare(true);
    402414      Algorithm.Start();
    403415      ExecutionState = ExecutionState.Started;
    … …  
    417429        throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState));
    418430      forceStop = true;
    419       Algorithm.Stop();
     431      try {
     432        Algorithm.Stop();
     433      } catch (InvalidOperationException) {
     434        // sometimes we hit the algorithm in an invalid state
     435      }
    420436    }
    421437
    … …  
    522538    }
    523539    private void Algorithm_Stopped(object sender, EventArgs e) {
    524       var bestQuality = UpdateAlgorithmResults();
     540      ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
     541      lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
    525542
    526543      var execTime = ((TimeSpanValue)CurrentRun.Results[ExecutionTimeResultName]).Value;
    527       double evaluationsInThisRun = 0;
    528       foreach (var result in Algorithm.Results) {
    529         if (result.Name == perClockAnalyzer.QualityPerClockParameter.ResultName) {
    530           if (!CurrentRun.Results.ContainsKey(result.Name))
    531             CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
    532           else {
    533             var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
    534             var best = dt.Rows.First().Values.Last().Item2;
    535             var resultDt = (IndexedDataTable<double>)result.Value;
    536             foreach (var tupl in resultDt.Rows.First().Values) {
    537               if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
    538                 dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2));
    539                 best = tupl.Item2;
    540               }
    541             }
     544      var solEvals = ((DoubleValue)CurrentRun.Results[EvaluatedSolutionsResultName]).Value;
     545      var movEvals = ((DoubleValue)CurrentRun.Results[EvaluatedMovesResultName]).Value;
     546      var restarts = ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value;
     547      var improvement = false;
     548
     549      IResult result;
     550      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out result)) {
     551        var evals = ((IntValue)result.Value).Value;
     552        Evaluations += evals;
     553        CurrentRun.Results[EvaluatedSolutionsResultName] = new DoubleValue(solEvals + evals);
     554      }
     555      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out result)) {
     556        var evals = ((IntValue)result.Value).Value;
     557        Evaluations += moveCostPerSolution * evals;
     558        CurrentRun.Results[EvaluatedMovesResultName] = new DoubleValue(movEvals + evals);
     559      }
     560      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out result)) {
     561        var bestQuality = ((DoubleValue)result.Value).Value;
     562        if (double.IsNaN(BestSoFar)
     563            || Maximization && bestQuality > BestSoFar
     564            || !Maximization && bestQuality < BestSoFar) {
     565          BestSoFar = bestQuality;
     566          CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
     567          improvement = true;
     568        }
     569      }
     570      if (Algorithm.Results.TryGetValue(perClockAnalyzer.QualityPerClockParameter.ResultName, out result)) {
     571        UpdateQualityPerClockResult((IndexedDataTable<double>)result.Value, execTime, restarts);
     572      }
     573      if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName, out result)) {
     574        UpdateQualityPerEvaluationsResult((IndexedDataTable<double>)result.Value, solEvals, movEvals, restarts);
     575      }
     576      if (StoreSolutionInRun) {
     577        foreach (var r in Algorithm.Results) {
     578          if (r.Name.ToLower().EndsWith("solution") && improvement) {
     579            CurrentRun.Results[r.Name] = (IItem)r.Value.Clone();
    542580          }
    543         } else if (result.Name == perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName) {
    544           if (!CurrentRun.Results.ContainsKey(result.Name))
    545             CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
    546           else {
    547             var dt = (IndexedDataTable<double>)CurrentRun.Results[result.Name];
    548             var evalSols = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
    549             var evalMoves = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
    550             var best = dt.Rows.First().Values.Last().Item2;
    551             var resultDt = (IndexedDataTable<double>)result.Value;
    552             foreach (var tupl in resultDt.Rows.First().Values) {
    553               if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
    554                 dt.Rows.First().Values.Add(Tuple.Create(evalSols + moveCostPerSolution * evalMoves + tupl.Item1, tupl.Item2));
    555                 best = tupl.Item2;
    556               }
    557             }
    558           }
    559         } else if (result.Name == perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName) {
    560           var oldEvals = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName]).Value;
    561           var newEvals = ((IntValue)result.Value).Value;
    562           CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName] = new IntValue(oldEvals + newEvals);
    563           evaluationsInThisRun += newEvals;
    564         } else if (result.Name == perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName) {
    565           var oldEvals = ((IntValue)CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName]).Value;
    566           var newEvals = ((IntValue)result.Value).Value;
    567           CurrentRun.Results[perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName] = new IntValue(oldEvals + newEvals);
    568           evaluationsInThisRun += newEvals * moveCostPerSolution;
    569         } else if (result.Name == perEvaluationsAnalyzer.BestQualityParameter.ActualName) {
    570           var best = ((DoubleValue)CurrentRun.Results[BestQualityResultName]).Value;
    571           if (Maximization && best < bestQuality || !Maximization && best > bestQuality)
    572             CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality);
    573         } else if (result.Name.ToLower().EndsWith("solution") && BestSoFar == bestQuality) {
    574           if (CurrentRun.Results.ContainsKey(result.Name))
    575             CurrentRun.Results[result.Name] = (IItem)result.Value.Clone();
    576           else CurrentRun.Results.Add(result.Name, (IItem)result.Value.Clone());
    577581        }
    578582      }
     583
    579584      CurrentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime);
    580585
    … …  
    584589
    585590      if (!forceStop && !IsFinished) {
    586         var restarts = ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value;
    587         if (restarts == 0) {
    588           CurrentRun.Results[RTusResultName] = new DoubleValue(Algorithm.ExecutionTime.TotalSeconds);
    589           CurrentRun.Results[FEusResultName] = new DoubleValue(evaluationsInThisRun);
    590         } else {
    591           var rtus = ((DoubleValue)CurrentRun.Results[RTusResultName]).Value;
    592           var feus = ((DoubleValue)CurrentRun.Results[FEusResultName]).Value;
    593           CurrentRun.Results[RTusResultName] = new DoubleValue(rtus * restarts / (restarts + 1.0) + Algorithm.ExecutionTime.TotalSeconds / (restarts + 1.0));
    594           CurrentRun.Results[FEusResultName] = new DoubleValue(feus * restarts / (restarts + 1.0) + evaluationsInThisRun / (restarts + 1.0));
    595         }
    596591        CurrentRun.Results[RandomRestartsResultName] = new IntValue(restarts + 1);
    597592        Algorithm.Prepare(true);
    598593        Algorithm.Start();
    599594      } else {
    600         if (Maximization && BestSoFar >= TargetValue || !Maximization && BestSoFar <= TargetValue) {
    601           CurrentRun.Results[RTsResultName] = (IItem)Algorithm.Results[perClockAnalyzer.QualityPerClockParameter.ResultName].Value.Clone();
    602           CurrentRun.Results[FEsResultName] = (IItem)Algorithm.Results[perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName].Value.Clone();
    603         }
    604595        forceStop = false;
    605596        Runs.Add(CurrentRun);
    … …  
    610601    }
    611602
    612     private double UpdateAlgorithmResults() {
    613       ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime;
    614       lastAlgorithmExecutionTime = Algorithm.ExecutionTime;
    615 
    616       IResult evaluationsResult;
    617       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out evaluationsResult)) {
    618         var evals = ((IntValue)evaluationsResult.Value).Value;
    619         Evaluations += evals;
    620       }
    621       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out evaluationsResult)) {
    622         var evals = ((IntValue)evaluationsResult.Value).Value;
    623         Evaluations += moveCostPerSolution * evals;
    624       }
    625       if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out evaluationsResult)) {
    626         var bestQuality = ((DoubleValue)evaluationsResult.Value).Value;
    627         if (double.IsNaN(BestSoFar)
    628             || Maximization && bestQuality > BestSoFar
    629             || !Maximization && bestQuality < BestSoFar)
    630           BestSoFar = bestQuality;
    631         return bestQuality;
    632       }
    633       return double.NaN;
     603    private void UpdateQualityPerClockResult(IndexedDataTable<double> perClock, TimeSpan execTime, int restarts) {
     604      IndexedDataTable<double> dt;
     605      if (!CurrentRun.Results.ContainsKey(QualityPerClockResultName)) {
     606        dt = (IndexedDataTable<double>)perClock.Clone();
     607        if (!dt.Rows.ContainsKey("Restarts"))
     608          dt.Rows.Add(new IndexedDataRow<double>("Restarts") {
     609            VisualProperties = {
     610              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
     611              SecondYAxis = true
     612            }
     613          });
     614        foreach (var v in dt.Rows.First().Values)
     615          dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0));
     616        CurrentRun.Results.Add(QualityPerClockResultName, dt);
     617      } else {
     618        dt = (IndexedDataTable<double>)CurrentRun.Results[QualityPerClockResultName];
     619        var best = dt.Rows.First().Values.Last().Item2;
     620        foreach (var tupl in perClock.Rows.First().Values) {
     621          if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
     622            dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2));
     623            dt.Rows["Restarts"].Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, (double)restarts));
     624            best = tupl.Item2;
     625          }
     626        }
     627      }
     628      if (IsFinished) {
     629        dt.Rows.First().Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, BestSoFar));
     630        dt.Rows["Restarts"].Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, (double)restarts));
     631      }
     632    }
     633
     634    private void UpdateQualityPerEvaluationsResult(IndexedDataTable<double> perEvaluations, double solEvals, double movEvals, int restarts) {
     635      IndexedDataTable<double> dt;
     636      if (!CurrentRun.Results.ContainsKey(QualityPerEvaluationsResultName)) {
     637        dt = (IndexedDataTable<double>)perEvaluations.Clone();
     638        if (!dt.Rows.ContainsKey("Restarts"))
     639          dt.Rows.Add(new IndexedDataRow<double>("Restarts") {
     640            VisualProperties = {
     641              ChartType = DataRowVisualProperties.DataRowChartType.StepLine,
     642              SecondYAxis = true
     643            }
     644          });
     645        foreach (var v in dt.Rows.First().Values)
     646          dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0));
     647        CurrentRun.Results.Add(QualityPerEvaluationsResultName, dt);
     648      } else {
     649        dt = (IndexedDataTable<double>)CurrentRun.Results[QualityPerEvaluationsResultName];
     650        var best = dt.Rows.First().Values.Last().Item2;
     651        foreach (var tupl in perEvaluations.Rows.First().Values) {
     652          if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) {
     653            dt.Rows.First().Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, tupl.Item2));
     654            dt.Rows["Restarts"].Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, (double)restarts));
     655            best = tupl.Item2;
     656          }
     657        }
     658      }
     659      if (IsFinished) {
     660        dt.Rows.First().Values.Add(Tuple.Create(Evaluations, BestSoFar));
     661        dt.Rows["Restarts"].Values.Add(Tuple.Create(Evaluations, (double)restarts));
     662      }
    634663    }
    635664
Note: See TracChangeset for help on using the changeset viewer.