#region License Information /* HeuristicLab * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL) * * This file is part of HeuristicLab. * * HeuristicLab is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * HeuristicLab is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with HeuristicLab. If not, see . */ #endregion using System; using System.Collections.Generic; using System.ComponentModel; using System.Drawing; using System.Linq; using HeuristicLab.Common; using HeuristicLab.Common.Resources; using HeuristicLab.Core; using HeuristicLab.Data; using HeuristicLab.Optimization; using HeuristicLab.Persistence.Default.CompositeSerializers.Storable; namespace HeuristicLab.Analysis { public enum TerminationCriterium { OnlyByTime, OnlyByEvaluations, OnlyByTarget, ByTargetAndTime, ByTargetAndEvaluations, ByTimeAndEvaluations, WhicheverHitsFirst, WhicheverHitsLast } /// /// A run in which an algorithm is executed for a certain maximum time only. /// [Item("Independent Random Restarter", "An optimizer that repeats an algorithm until either a certain target value is reached or a maximum budget is exceeded.")] [Creatable(CreatableAttribute.Categories.TestingAndAnalysis, Priority = 117)] [StorableClass] public sealed class IndepdentRandomRestarter : NamedItem, IOptimizer, IStorableContent, INotifyPropertyChanged { private const string ExecutionTimeResultName = "Execution Time"; private const string BestQualityResultName = "BestQuality"; private const string RandomRestartsResultName = "RandomRestarts"; private const string EvaluatedSolutionsResultName = "EvaluatedSolutions"; private const string EvaluatedMovesResultName = "EvaluatedMoves"; private const string QualityPerClockResultName = "QualityPerClock"; private const string QualityPerEvaluationsResultName = "QualityPerEvaluations"; public string Filename { get; set; } #region ItemImage public static new Image StaticItemImage { get { return VSImageLibrary.Event; } } public override Image ItemImage { get { return (Algorithm != null) ? Algorithm.ItemImage : VSImageLibrary.ExecutableStopped; } } #endregion [Storable] private TerminationCriterium terminationCriterium; public TerminationCriterium TerminationCriterium { get { return terminationCriterium; } set { if (terminationCriterium == value) return; terminationCriterium = value; OnPropertyChanged("TerminationCriterium"); } } [Storable] private TimeSpan maximumExecutionTime; public TimeSpan MaximumExecutionTime { get { return maximumExecutionTime; } set { if (maximumExecutionTime == value) return; maximumExecutionTime = value; OnPropertyChanged("MaximumExecutionTime"); } } [Storable] private int maximumEvaluations; public int MaximumEvaluations { get { return maximumEvaluations; } set { if (maximumEvaluations == value) return; maximumEvaluations = value; OnPropertyChanged("MaximumEvaluations"); } } [Storable] private double targetValue; public double TargetValue { get { return targetValue; } set { if (targetValue == value) return; targetValue = value; OnPropertyChanged("TargetValue"); } } [Storable] private bool maximization; public bool Maximization { get { return maximization; } set { if (maximization == value) return; maximization = value; OnPropertyChanged("Maximization"); } } [Storable] private double moveCostPerSolution; public double MoveCostPerSolution { get { return moveCostPerSolution; } set { if (moveCostPerSolution == value) return; moveCostPerSolution = value; perEvaluationsAnalyzer.MoveCostPerSolutionParameter.Value = new DoubleValue(moveCostPerSolution); OnPropertyChanged("MoveCostPerSolution"); } } [Storable] private bool storeSolutionInRun; public bool StoreSolutionInRun { get { return storeSolutionInRun; } set { if (storeSolutionInRun == value) return; storeSolutionInRun = value; OnPropertyChanged("StoreSolutionInRun"); } } [Storable] private QualityPerClockAnalyzer perClockAnalyzer; [Storable] private QualityPerEvaluationsAnalyzer perEvaluationsAnalyzer; [Storable] private ExecutionState executionState; public ExecutionState ExecutionState { get { return executionState; } private set { if (executionState != value) { executionState = value; OnExecutionStateChanged(); OnItemImageChanged(); } } } private TimeSpan lastAlgorithmExecutionTime; [Storable] private TimeSpan executionTime; public TimeSpan ExecutionTime { get { return executionTime; } set { if (executionTime == value) return; executionTime = value; OnPropertyChanged("ExecutionTime"); OnExecutionTimeChanged(); } } [Storable] private double evaluations; public double Evaluations { get { return evaluations; } set { if (evaluations == value) return; evaluations = value; OnPropertyChanged("Evaluations"); } } [Storable] private double bestSoFar; public double BestSoFar { get { return bestSoFar; } set { if (bestSoFar == value) return; bestSoFar = value; OnPropertyChanged("BestSoFar"); } } [Storable] private IRun currentRun; public IRun CurrentRun { get { return currentRun; } private set { if (currentRun == value) return; currentRun = value; OnPropertyChanged("CurrentRun"); } } [Storable] private IAlgorithm algorithm; public IAlgorithm Algorithm { get { return algorithm; } set { if (algorithm == value) return; if (algorithm != null) { DeregisterAlgorithmEvents(); RemoveAlgorithmAnalyzers(); } algorithm = value; if (algorithm != null) { RegisterAlgorithmEvents(); AddAlgorithmAnalyzers(); } OnPropertyChanged("Algorithm"); Prepare(); } } [Storable] private RunCollection runs; public RunCollection Runs { get { return runs; } private set { if (value == null) throw new ArgumentNullException(); if (runs == value) return; runs = value; OnPropertyChanged("Runs"); } } public IEnumerable NestedOptimizers { get { if (Algorithm == null) yield break; yield return Algorithm; foreach (var opt in Algorithm.NestedOptimizers) yield return opt; } } private bool IsFinished { get { var timeHit = ExecutionTime >= MaximumExecutionTime; var evalHit = Evaluations >= MaximumEvaluations; var targetHit = (Maximization && BestSoFar >= TargetValue || !Maximization && BestSoFar <= TargetValue); return timeHit && evalHit && targetHit || timeHit && (TerminationCriterium == TerminationCriterium.OnlyByTime || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst || TerminationCriterium == TerminationCriterium.ByTargetAndTime || TerminationCriterium == TerminationCriterium.ByTimeAndEvaluations) || evalHit && (TerminationCriterium == TerminationCriterium.OnlyByEvaluations || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst || TerminationCriterium == TerminationCriterium.ByTargetAndEvaluations || TerminationCriterium == TerminationCriterium.ByTimeAndEvaluations) || targetHit && (TerminationCriterium == TerminationCriterium.OnlyByTarget || TerminationCriterium == TerminationCriterium.WhicheverHitsFirst || TerminationCriterium == TerminationCriterium.ByTargetAndTime || TerminationCriterium == TerminationCriterium.ByTargetAndEvaluations); } } private ISingleObjectiveHeuristicOptimizationProblem problem; [StorableConstructor] private IndepdentRandomRestarter(bool deserializing) : base(deserializing) { } private IndepdentRandomRestarter(IndepdentRandomRestarter original, Cloner cloner) : base(original, cloner) { terminationCriterium = original.terminationCriterium; maximumExecutionTime = original.maximumExecutionTime; maximumEvaluations = original.maximumEvaluations; moveCostPerSolution = original.moveCostPerSolution; storeSolutionInRun = original.storeSolutionInRun; targetValue = original.targetValue; maximization = original.maximization; executionTime = original.executionTime; evaluations = original.evaluations; bestSoFar = original.bestSoFar; lastAlgorithmExecutionTime = original.lastAlgorithmExecutionTime; perClockAnalyzer = cloner.Clone(original.perClockAnalyzer); perEvaluationsAnalyzer = cloner.Clone(original.perEvaluationsAnalyzer); algorithm = cloner.Clone(original.algorithm); runs = cloner.Clone(original.runs); ExecutionState = original.ExecutionState; Initialize(); } public IndepdentRandomRestarter() : base() { name = ItemName; description = ItemDescription; terminationCriterium = TerminationCriterium.ByTargetAndEvaluations; maximumExecutionTime = TimeSpan.FromMinutes(1); maximumEvaluations = 10000000; // 10 mio moveCostPerSolution = 1; storeSolutionInRun = false; targetValue = 0; maximization = false; executionTime = TimeSpan.Zero; evaluations = 0; bestSoFar = double.NaN; lastAlgorithmExecutionTime = TimeSpan.Zero; perClockAnalyzer = new QualityPerClockAnalyzer(); perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer(); Runs = new RunCollection { OptimizerName = Name }; Initialize(); } public IndepdentRandomRestarter(string name) : base(name) { description = ItemDescription; terminationCriterium = TerminationCriterium.ByTargetAndEvaluations; maximumExecutionTime = TimeSpan.FromMinutes(1); maximumEvaluations = 10000000; // 10 mio moveCostPerSolution = 1; storeSolutionInRun = false; targetValue = 0; maximization = false; executionTime = TimeSpan.Zero; evaluations = 0; bestSoFar = double.NaN; lastAlgorithmExecutionTime = TimeSpan.Zero; perClockAnalyzer = new QualityPerClockAnalyzer(); perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer(); Runs = new RunCollection { OptimizerName = Name }; Initialize(); } public IndepdentRandomRestarter(string name, string description) : base(name, description) { terminationCriterium = TerminationCriterium.ByTargetAndEvaluations; maximumExecutionTime = TimeSpan.FromMinutes(1); maximumEvaluations = 10000000; // 10 mio moveCostPerSolution = 1; storeSolutionInRun = false; targetValue = 0; maximization = false; executionTime = TimeSpan.Zero; evaluations = 0; bestSoFar = double.NaN; lastAlgorithmExecutionTime = TimeSpan.Zero; perClockAnalyzer = new QualityPerClockAnalyzer(); perEvaluationsAnalyzer = new QualityPerEvaluationsAnalyzer(); Runs = new RunCollection { OptimizerName = Name }; Initialize(); } public override IDeepCloneable Clone(Cloner cloner) { if (ExecutionState == ExecutionState.Started) throw new InvalidOperationException(string.Format("Clone not allowed in execution state \"{0}\".", ExecutionState)); return new IndepdentRandomRestarter(this, cloner); } [StorableHook(HookType.AfterDeserialization)] private void AfterDeserialization() { Initialize(); } private void Initialize() { if (algorithm != null) RegisterAlgorithmEvents(); } private void Reset() { ExecutionTime = TimeSpan.Zero; Evaluations = 0; BestSoFar = double.NaN; lastAlgorithmExecutionTime = TimeSpan.Zero; CurrentRun = null; } public void Prepare() { Prepare(false); } public void Prepare(bool clearRuns) { if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused) && (ExecutionState != ExecutionState.Stopped)) throw new InvalidOperationException(string.Format("Prepare not allowed in execution state \"{0}\".", ExecutionState)); Reset(); if (Algorithm != null) { Algorithm.Prepare(clearRuns); ExecutionState = ExecutionState.Prepared; OnPrepared(); } } public void Start() { if ((ExecutionState != ExecutionState.Prepared) && (ExecutionState != ExecutionState.Paused)) throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState)); if (ExecutionState == ExecutionState.Prepared) { CurrentRun = new Run(Algorithm) { Name = Algorithm.Name + " IRRRun" + Runs.Count }; if (!CurrentRun.Results.ContainsKey(ExecutionTimeResultName)) CurrentRun.Results.Add(ExecutionTimeResultName, new TimeSpanValue(TimeSpan.Zero)); // use double instead of int, otherwise one might run into int.MaxValue (at least with moves) CurrentRun.Results.Add(EvaluatedSolutionsResultName, new DoubleValue(0)); CurrentRun.Results.Add(EvaluatedMovesResultName, new DoubleValue(0)); CurrentRun.Results.Add(BestQualityResultName, new DoubleValue(Maximization ? double.MinValue : double.MaxValue)); CurrentRun.Results.Add(RandomRestartsResultName, new IntValue(0)); } if (Algorithm.ExecutionState == ExecutionState.Stopped) Algorithm.Prepare(true); Algorithm.Start(); ExecutionState = ExecutionState.Started; OnStarted(); } public void Pause() { if (ExecutionState != ExecutionState.Started) throw new InvalidOperationException(string.Format("Pause not allowed in execution state \"{0}\".", ExecutionState)); Algorithm.Pause(); ExecutionState = ExecutionState.Paused; OnPaused(); } private bool forceStop = false; public void Stop() { if ((ExecutionState != ExecutionState.Started) && (ExecutionState != ExecutionState.Paused)) throw new InvalidOperationException(string.Format("Stop not allowed in execution state \"{0}\".", ExecutionState)); forceStop = true; try { Algorithm.Stop(); } catch (InvalidOperationException) { // sometimes we hit the algorithm in an invalid state } } private void AddAlgorithmAnalyzers() { if (!Algorithm.Parameters.ContainsKey("Analyzer")) return; var analyzerParam = Algorithm.Parameters["Analyzer"] as IValueParameter; if (analyzerParam == null) return; if (!analyzerParam.Value.Operators.Contains(perClockAnalyzer)) analyzerParam.Value.Operators.Add(perClockAnalyzer); if (!analyzerParam.Value.Operators.Contains(perEvaluationsAnalyzer)) analyzerParam.Value.Operators.Add(perEvaluationsAnalyzer); } private void RemoveAlgorithmAnalyzers() { if (!Algorithm.Parameters.ContainsKey("Analyzer")) return; var analyzerParam = Algorithm.Parameters["Analyzer"] as IValueParameter; if (analyzerParam == null) return; analyzerParam.Value.Operators.Remove(perClockAnalyzer); analyzerParam.Value.Operators.Remove(perEvaluationsAnalyzer); } #region Events protected override void OnNameChanged() { base.OnNameChanged(); runs.OptimizerName = Name; } public event PropertyChangedEventHandler PropertyChanged; private void OnPropertyChanged(string property) { var handler = PropertyChanged; if (handler != null) handler(this, new PropertyChangedEventArgs(property)); } #region IExecutable Events public event EventHandler ExecutionStateChanged; private void OnExecutionStateChanged() { var handler = ExecutionStateChanged; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler ExecutionTimeChanged; private void OnExecutionTimeChanged() { var handler = ExecutionTimeChanged; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler Prepared; private void OnPrepared() { var handler = Prepared; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler Started; private void OnStarted() { var handler = Started; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler Paused; private void OnPaused() { var handler = Paused; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler Stopped; private void OnStopped() { var handler = Stopped; if (handler != null) handler(this, EventArgs.Empty); } public event EventHandler> ExceptionOccurred; private void OnExceptionOccurred(Exception exception) { var handler = ExceptionOccurred; if (handler != null) handler(this, new EventArgs(exception)); } #endregion #region Algorithm Events private void RegisterAlgorithmEvents() { algorithm.ExceptionOccurred += Algorithm_ExceptionOccurred; algorithm.ExecutionTimeChanged += Algorithm_ExecutionTimeChanged; algorithm.Paused += Algorithm_Paused; algorithm.Prepared += Algorithm_Prepared; algorithm.Stopped += Algorithm_Stopped; algorithm.ProblemChanged += Algorithm_ProblemChanged; Algorithm_ProblemChanged(algorithm, EventArgs.Empty); } private void DeregisterAlgorithmEvents() { algorithm.ExceptionOccurred -= Algorithm_ExceptionOccurred; algorithm.ExecutionTimeChanged -= Algorithm_ExecutionTimeChanged; algorithm.Paused -= Algorithm_Paused; algorithm.Prepared -= Algorithm_Prepared; algorithm.Stopped -= Algorithm_Stopped; algorithm.ProblemChanged -= Algorithm_ProblemChanged; } private void Algorithm_ExceptionOccurred(object sender, EventArgs e) { OnExceptionOccurred(e.Value); } private void Algorithm_ExecutionTimeChanged(object sender, EventArgs e) { ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime; lastAlgorithmExecutionTime = Algorithm.ExecutionTime; } private void Algorithm_Paused(object sender, EventArgs e) { ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime; lastAlgorithmExecutionTime = Algorithm.ExecutionTime; OnPaused(); } private void Algorithm_Prepared(object sender, EventArgs e) { lastAlgorithmExecutionTime = TimeSpan.Zero; } private void Algorithm_Stopped(object sender, EventArgs e) { ExecutionTime += Algorithm.ExecutionTime - lastAlgorithmExecutionTime; lastAlgorithmExecutionTime = Algorithm.ExecutionTime; var execTime = ((TimeSpanValue)CurrentRun.Results[ExecutionTimeResultName]).Value; var solEvals = ((DoubleValue)CurrentRun.Results[EvaluatedSolutionsResultName]).Value; var movEvals = ((DoubleValue)CurrentRun.Results[EvaluatedMovesResultName]).Value; var restarts = ((IntValue)CurrentRun.Results[RandomRestartsResultName]).Value; var improvement = false; IResult result; if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedSolutionsParameter.ActualName, out result)) { var evals = ((IntValue)result.Value).Value; Evaluations += evals; CurrentRun.Results[EvaluatedSolutionsResultName] = new DoubleValue(solEvals + evals); } if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.EvaluatedMovesParameter.ActualName, out result)) { var evals = ((IntValue)result.Value).Value; Evaluations += moveCostPerSolution * evals; CurrentRun.Results[EvaluatedMovesResultName] = new DoubleValue(movEvals + evals); } if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.BestQualityParameter.ActualName, out result)) { var bestQuality = ((DoubleValue)result.Value).Value; if (double.IsNaN(BestSoFar) || Maximization && bestQuality > BestSoFar || !Maximization && bestQuality < BestSoFar) { BestSoFar = bestQuality; CurrentRun.Results[BestQualityResultName] = new DoubleValue(bestQuality); improvement = true; } } if (Algorithm.Results.TryGetValue(perClockAnalyzer.QualityPerClockParameter.ResultName, out result)) { UpdateQualityPerClockResult((IndexedDataTable)result.Value, execTime, restarts); } if (Algorithm.Results.TryGetValue(perEvaluationsAnalyzer.QualityPerEvaluationsParameter.ResultName, out result)) { UpdateQualityPerEvaluationsResult((IndexedDataTable)result.Value, solEvals, movEvals, restarts); } if (StoreSolutionInRun) { foreach (var r in Algorithm.Results) { if (r.Name.ToLower().EndsWith("solution") && improvement) { CurrentRun.Results[r.Name] = (IItem)r.Value.Clone(); } } } CurrentRun.Results[ExecutionTimeResultName] = new TimeSpanValue(execTime + Algorithm.ExecutionTime); // Algorithm sets ExecutionTime to zero before firing Prepared // We will thus see ExecutionTimeChanged before Prepared lastAlgorithmExecutionTime = TimeSpan.Zero; if (!forceStop && !IsFinished) { CurrentRun.Results[RandomRestartsResultName] = new IntValue(restarts + 1); Algorithm.Prepare(true); Algorithm.Start(); } else { forceStop = false; Runs.Add(CurrentRun); Algorithm.Prepare(true); ExecutionState = ExecutionState.Stopped; OnStopped(); } } private void UpdateQualityPerClockResult(IndexedDataTable perClock, TimeSpan execTime, int restarts) { IndexedDataTable dt; if (!CurrentRun.Results.ContainsKey(QualityPerClockResultName)) { dt = (IndexedDataTable)perClock.Clone(); if (!dt.Rows.ContainsKey("Restarts")) dt.Rows.Add(new IndexedDataRow("Restarts") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine, SecondYAxis = true } }); foreach (var v in dt.Rows.First().Values) dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0)); CurrentRun.Results.Add(QualityPerClockResultName, dt); } else { dt = (IndexedDataTable)CurrentRun.Results[QualityPerClockResultName]; var best = dt.Rows.First().Values.Last().Item2; foreach (var tupl in perClock.Rows.First().Values) { if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) { dt.Rows.First().Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, tupl.Item2)); dt.Rows["Restarts"].Values.Add(Tuple.Create(execTime.TotalSeconds + tupl.Item1, (double)restarts)); best = tupl.Item2; } } } if (IsFinished) { dt.Rows.First().Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, BestSoFar)); dt.Rows["Restarts"].Values.Add(Tuple.Create(ExecutionTime.TotalSeconds, (double)restarts)); } } private void UpdateQualityPerEvaluationsResult(IndexedDataTable perEvaluations, double solEvals, double movEvals, int restarts) { IndexedDataTable dt; if (!CurrentRun.Results.ContainsKey(QualityPerEvaluationsResultName)) { dt = (IndexedDataTable)perEvaluations.Clone(); if (!dt.Rows.ContainsKey("Restarts")) dt.Rows.Add(new IndexedDataRow("Restarts") { VisualProperties = { ChartType = DataRowVisualProperties.DataRowChartType.StepLine, SecondYAxis = true } }); foreach (var v in dt.Rows.First().Values) dt.Rows["Restarts"].Values.Add(Tuple.Create(v.Item1, 0.0)); CurrentRun.Results.Add(QualityPerEvaluationsResultName, dt); } else { dt = (IndexedDataTable)CurrentRun.Results[QualityPerEvaluationsResultName]; var best = dt.Rows.First().Values.Last().Item2; foreach (var tupl in perEvaluations.Rows.First().Values) { if (Maximization && tupl.Item2 > best || !Maximization && tupl.Item2 < best) { dt.Rows.First().Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, tupl.Item2)); dt.Rows["Restarts"].Values.Add(Tuple.Create(solEvals + moveCostPerSolution * movEvals + tupl.Item1, (double)restarts)); best = tupl.Item2; } } } if (IsFinished) { dt.Rows.First().Values.Add(Tuple.Create(Evaluations, BestSoFar)); dt.Rows["Restarts"].Values.Add(Tuple.Create(Evaluations, (double)restarts)); } } private void Algorithm_ProblemChanged(object sender, EventArgs e) { if (problem != null) DeregisterProblemEvents(); problem = Algorithm.Problem as ISingleObjectiveHeuristicOptimizationProblem; if (problem == null) return; RegisterProblemEvents(); AddAlgorithmAnalyzers(); var maxParam = problem.MaximizationParameter as IValueParameter; if (maxParam != null) Maximization = maxParam.Value.Value; var bkParam = problem.BestKnownQualityParameter as IValueParameter; if (bkParam != null && bkParam.Value != null) TargetValue = bkParam.Value.Value; Reset(); } private void RegisterProblemEvents() { problem.Reset += ProblemOnReset; problem.OperatorsChanged += ProblemOnOperatorsChanged; } private void DeregisterProblemEvents() { problem.Reset -= ProblemOnReset; problem.OperatorsChanged -= ProblemOnOperatorsChanged; } private void ProblemOnReset(object sender, EventArgs eventArgs) { AddAlgorithmAnalyzers(); } private void ProblemOnOperatorsChanged(object sender, EventArgs eventArgs) { AddAlgorithmAnalyzers(); } #endregion #endregion } }