using System; using System.Linq; using System.Collections.Generic; using HeuristicLab.Common; using HeuristicLab.Core; using HeuristicLab.Data; using HeuristicLab.Optimization; using HeuristicLab.Parameters; using HeuristicLab.Persistence.Default.CompositeSerializers.Storable; using HeuristicLab.PluginInfrastructure; using HeuristicLab.Encodings.BinaryVectorEncoding; using HeuristicLab.Encodings.RealVectorEncoding; using HeuristicLab.Encodings.PermutationEncoding; using HeuristicLab.Common.Resources; using System.Drawing; namespace HeuristicLab.Analysis.FitnessLandscape.MultiTrajectory { [Item("Multi Trajectory Analysis", "Host for performing and aggregating several algorithm runs for an overarching analysis.")] [Creatable("Analysis")] [StorableClass] public class MultiTrajectoryAnalysis : ParameterizedNamedItem, IAlgorithm, IStorableContent { public IEnumerable NestedOptimizers { get { if (Algorithm != null) yield return Algorithm; if (SampleGenerationAlgorithm != null) yield return SampleGenerationAlgorithm; } } #region Parameters public OperatorParameter PreassignedSolutionCreatorParameter { get { return (OperatorParameter)Parameters["PreassignedSolutionCreator"]; } } public ValueParameter NrOfSamplesParameter { get { return (ValueParameter)Parameters["NrOfSamples"]; } } public ValueParameter SaveIndividualTrailsParameter { get { return (ValueParameter)Parameters["SaveIndividualTrails"]; } } public ValueParameter PrepareAfterStopParameter { get { return (ValueParameter)Parameters["PrepareAfterStop"]; } } public ValueParameter> AggregatorsParameter { get { return (ValueParameter>)Parameters["Aggregators"]; } } #endregion #region Parameter Values public PreassignedSolutionCreator PreassignedSolutionCreator { get { return (PreassignedSolutionCreator)PreassignedSolutionCreatorParameter.Value; } set { PreassignedSolutionCreatorParameter.Value = value; } } public IntValue NrOfSamples { get { return NrOfSamplesParameter.Value; } } public bool SaveIndividualTrails { get { return SaveIndividualTrailsParameter.Value.Value; } } public bool PrepareAfterStop { get { return PrepareAfterStopParameter.Value.Value; } } public CheckedItemList Aggregators { get { return AggregatorsParameter.Value; } } #endregion #region Construction and Cloning [StorableConstructor] protected MultiTrajectoryAnalysis(bool deserializing) : base(deserializing) { RegisterEvents(); } protected MultiTrajectoryAnalysis(MultiTrajectoryAnalysis original, Cloner cloner) : base(original, cloner) { algorithm = cloner.Clone(original.algorithm); currentAlgorithm = cloner.Clone(original.currentAlgorithm); executionBaseTime = new TimeSpan(original.executionBaseTime.Ticks); executionTime = new TimeSpan(original.executionTime.Ticks); results = cloner.Clone(original.results); running = original.running; preparing = original.preparing; runs = cloner.Clone(original.runs); sampleGenerationAlgorithm = cloner.Clone(original.sampleGenerationAlgorithm); StepNr = original.StepNr; storeAlgorithmInEachRun = original.storeAlgorithmInEachRun; subAlgorithmRuns = cloner.Clone(original.subAlgorithmRuns); RegisterEvents(); } public MultiTrajectoryAnalysis() { Parameters.Add(new OperatorParameter("PreassignedSolutionCreator", "The operator that creates new solutions")); Parameters.Add(new ValueParameter("NrOfSamples", "The number of samples to generate.", new IntValue(10))); Parameters.Add(new ValueParameter("SaveIndividualTrails", "Include all sub algorithm runs in the final run", new BoolValue(true))); Parameters.Add(new ValueParameter("PrepareAfterStop", "Automatically prepare algorithm after stopping to clean global scope and results (for large experiments)", new BoolValue(false))); Parameters.Add(new ValueParameter>("Aggregators", "List of aggretators for combining trajectory results.", new CheckedItemList())); currentAlgorithm = null; algorithm = null; executionBaseTime = new TimeSpan(); executionTime = new TimeSpan(); results = new ResultCollection(); running = false; runs = new RunCollection(); sampleGenerationAlgorithm = null; StepNr = -1; storeAlgorithmInEachRun = false; Aggregators.AddRange(ApplicationManager.Manager.GetInstances()); RegisterEvents(); Prepare(); } public override IDeepCloneable Clone(Cloner cloner) { return new MultiTrajectoryAnalysis(this, cloner); } [StorableHook(HookType.AfterDeserialization)] private void AfterDeserialization() { if (algorithm != null && sampleGenerationAlgorithm != null) { if (sampleGenerationAlgorithm.Problem == null) { sampleGenerationAlgorithm.Problem = algorithm.Problem; } else { algorithm.Problem = sampleGenerationAlgorithm.Problem; } } if (!Parameters.ContainsKey("Aggregators")) Parameters.Add(new ValueParameter>( "Aggregators", "List of aggregators for combining trajectory results.", new CheckedItemList(ApplicationManager.Manager.GetInstances()))); RegisterEvents(); } #endregion #region Fields & Properties public new static Image StaticItemImage { get { return VSImageLibrary.Event; } } public override Image ItemImage { get { switch (ExecutionState) { case ExecutionState.Prepared: return VSImageLibrary.ExecutablePrepared; case ExecutionState.Started: return VSImageLibrary.ExecutableStarted; case ExecutionState.Paused: return VSImageLibrary.ExecutablePaused; case ExecutionState.Stopped: return VSImageLibrary.ExecutableStopped; default: return base.ItemImage; } } } [Storable] private RunCollection subAlgorithmRuns; public new ParameterCollection Parameters { get { return base.Parameters; } } public Type ProblemType { get { return typeof(IHeuristicOptimizationProblem); } } public string Filename { get; set; } [Storable] private IProblem preliminaryProblem; public IProblem Problem { get { return preliminaryProblem ?? algorithm.Problem; } set { if (ExecutionState != ExecutionState.Prepared && ExecutionState != ExecutionState.Stopped) throw new InvalidOperationException("Cannot change algorithm while running or paused."); if (algorithm == null) preliminaryProblem = value; else algorithm.Problem = value; } } [Storable] private IAlgorithm algorithm; public IAlgorithm Algorithm { get { return algorithm; } set { if (algorithm == value) return; if (algorithm != null) { DeregisterAlgorithmEvents(); ReuseOldProblem(algorithm, value); } algorithm = value; if (algorithm != null) { SyncAlgorithmProblems(algorithm, sampleGenerationAlgorithm); RegisterAlgorithmEvents(); } OnAlgorithmChanged(); } } [Storable] private IAlgorithm sampleGenerationAlgorithm; public IAlgorithm SampleGenerationAlgorithm { get { return sampleGenerationAlgorithm; } set { if (sampleGenerationAlgorithm == value) return; if (sampleGenerationAlgorithm != null) { DeregisterSampleGenerationAlgorithmEvents(); ReuseOldProblem(sampleGenerationAlgorithm, value); } sampleGenerationAlgorithm = value; if (sampleGenerationAlgorithm != null) { SyncAlgorithmProblems(sampleGenerationAlgorithm, algorithm); RegisterSampleGenerationAlgorithmEvents(); } OnSampleGenerationAlgorithmChanged(); } } private static void ReuseOldProblem(IAlgorithm oldAlgorithm, IAlgorithm newValue) { if (oldAlgorithm.Problem != null && newValue != null && newValue.Problem == null) newValue.Problem = oldAlgorithm.Problem; } private void SyncAlgorithmProblems(IAlgorithm newAlgorithm, IAlgorithm otherAlgorithm) { if (otherAlgorithm != null) { if (otherAlgorithm.Problem != null) { newAlgorithm.Problem = otherAlgorithm.Problem; } else { otherAlgorithm.Problem = newAlgorithm.Problem; } } else { if (preliminaryProblem != null) { newAlgorithm.Problem = preliminaryProblem; preliminaryProblem = null; } } } [Storable] private RunCollection runs; public RunCollection Runs { get { return runs; } } [Storable] private ResultCollection results; public ResultCollection Results { get { return results; } } [Storable] private bool storeAlgorithmInEachRun; public bool StoreAlgorithmInEachRun { get { return storeAlgorithmInEachRun; } set { if (storeAlgorithmInEachRun == value) return; storeAlgorithmInEachRun = value; OnStoreAlgorithmInEachRunChanged(); } } [Storable] private ExecutionState executionState; public ExecutionState ExecutionState { get { return executionState; } set { if (executionState == value) return; executionState = value; OnExecutionStateChanged(); OnItemImageChanged(); } } [Storable] private TimeSpan executionBaseTime; [Storable] private TimeSpan executionTime; public TimeSpan ExecutionTime { get { return executionTime; } set { if (executionTime == value) return; executionTime = value; OnExecutionTimeChanged(); } } #endregion #region Events public event EventHandler AlgorithmChanged; public event EventHandler SampleGenerationAlgorithmChanged; public event EventHandler ProblemChanged; public event EventHandler StoreAlgorithmInEachRunChanged; public event EventHandler> ExceptionOccurred; public event EventHandler ExecutionStateChanged; public event EventHandler ExecutionTimeChanged; public event EventHandler Paused; public event EventHandler Prepared; public event EventHandler Started; public event EventHandler Stopped; #region firing protected virtual void OnAlgorithmChanged() { EventHandler handler = AlgorithmChanged; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnSampleGenerationAlgorithmChanged() { EventHandler handler = SampleGenerationAlgorithmChanged; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnProblemChanged() { SelectPreassignedSolutionCreator(); EventHandler handler = ProblemChanged; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnStoreAlgorithmInEachRunChanged() { EventHandler handler = StoreAlgorithmInEachRunChanged; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnExceptionOccurred(Exception args) { EventHandler> handler = ExceptionOccurred; if (handler != null) handler(this, new EventArgs(args)); } protected virtual void OnExecutionStateChanged() { EventHandler handler = ExecutionStateChanged; if (handler != null) handler(this, EventArgs.Empty); switch (ExecutionState) { case ExecutionState.Prepared: OnPrepared(); break; case ExecutionState.Started: OnStarted(); break; case ExecutionState.Paused: OnPaused(); break; case ExecutionState.Stopped: OnStopped(); break; } } protected virtual void OnExecutionTimeChanged() { EventHandler handler = ExecutionTimeChanged; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnPaused() { EventHandler handler = Paused; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnPrepared() { EventHandler handler = Prepared; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnStarted() { EventHandler handler = Started; if (handler != null) handler(this, EventArgs.Empty); } protected virtual void OnStopped() { AggregateResults(); ClearSubAlgorithms(); CreateRun(); EventHandler handler = Stopped; if (handler != null) handler(this, EventArgs.Empty); if (PrepareAfterStop) { Prepare(); executionState = ExecutionState.Stopped; } } #endregion #endregion public void CollectResultValues(IDictionary values) { values.Add("Execution Time", new TimeSpanValue(ExecutionTime)); values.Add("Time of Execution", new DateTimeValue(DateTime.Now)); CollectInnerResults(values, "", results); } private static void CollectInnerResults(IDictionary values, string prefix, IEnumerable results) { foreach (var result in results) { var innerResults = result.Value as IEnumerable; if (innerResults == null) values.Add(string.Format("{0}{1}", prefix, result.Name), result.Value); else CollectInnerResults(values, string.Format("{0}{1}.", prefix, result.Name), innerResults); } } [Storable] bool running; [Storable] bool preparing; [Storable] private int StepNr; [Storable] IAlgorithm currentAlgorithm; public void Prepare(bool clearRuns) { if (clearRuns) Runs.Clear(); Prepare(); } public void Prepare() { try { preparing = true; try { if (Algorithm != null) { Algorithm.Prepare(); Algorithm.StoreAlgorithmInEachRun = false; } if (SampleGenerationAlgorithm != null) { SampleGenerationAlgorithm.Prepare(); SampleGenerationAlgorithm.StoreAlgorithmInEachRun = false; } ClearSubAlgorithms(); StepNr = -1; ExecutionTime = new TimeSpan(); executionBaseTime = new TimeSpan(); results.Clear(); subAlgorithmRuns = new RunCollection(); results.Add(new Result("Sub Algorithm Runs", subAlgorithmRuns)); if (PreassignedSolutionCreator != null) { if (PreassignedSolutionCreator.SolutionCollection != null) PreassignedSolutionCreator.SolutionCollection.Clear(); PreassignedSolutionCreator.CurrentSolutionIndex = -1; } ExecutionState = ExecutionState.Prepared; } catch (Exception) { } } finally { preparing = false; } } public void Pause() { try { running = false; if (currentAlgorithm != null) currentAlgorithm.Pause(); ExecutionState = ExecutionState.Paused; } catch (Exception) { } } public void Start() { running = true; ExecutionState = ExecutionState.Started; PerformNextStep(); } public void Stop() { running = false; try { if (currentAlgorithm != null) currentAlgorithm.Stop(); } catch (Exception) { } } protected virtual void PerformNextStep() { PreassignedSolutionCreator.CurrentSolutionIndex = StepNr; if (StepNr == -1) { currentAlgorithm = SampleGenerationAlgorithm; currentAlgorithm.Start(); } else if (StepNr < NrOfSamples.Value) { currentAlgorithm = Algorithm; IHeuristicOptimizationProblem problem = Algorithm.Problem as IHeuristicOptimizationProblem; if (problem != null) { SynchronizeParameters(problem.SolutionCreator, PreassignedSolutionCreator as ISolutionCreator); if (PreassignedSolutionCreator != problem.SolutionCreator) PreassignedSolutionCreator.OriginalSolutionCreator = problem.SolutionCreator; problem.SolutionCreatorParameter.ActualValue = PreassignedSolutionCreator; } Algorithm.Prepare(); Algorithm.Runs.Clear(); Algorithm.Start(); } else { ExecutionState = ExecutionState.Stopped; } } private void ClearSubAlgorithms() { if (sampleGenerationAlgorithm != null) { sampleGenerationAlgorithm.Results.Clear(); sampleGenerationAlgorithm.Runs.Clear(); } if (algorithm != null) { algorithm.Results.Clear(); algorithm.Runs.Clear(); } } private void AggregateResults() { if (subAlgorithmRuns == null) return; foreach (var aggregator in Aggregators) aggregator.Reset(); foreach (IRun run in subAlgorithmRuns) { foreach (var result in run.Results) { foreach (var item in Aggregators.CheckedItems) item.Value.MaybeAddResult(new Result(result.Key, result.Value)); } } foreach (var item in Aggregators.CheckedItems) results.Add(item.Value.CreateResult()); } public override void CollectParameterValues(IDictionary values) { base.CollectParameterValues(values); values.Add("Algorithm Name", new StringValue(Name)); values.Add("Algorithm Type", new StringValue(this.GetType().GetPrettyName())); if (Problem != null) { Problem.CollectParameterValues(values); values.Add("Problem Name", new StringValue(Problem.Name)); values.Add("Problem Type", new StringValue(Problem.GetType().GetPrettyName())); } CollectPrefixedParameterValues("Sampling", SampleGenerationAlgorithm, values); CollectPrefixedParameterValues("Analysis", Algorithm, values); } private void CollectPrefixedParameterValues(string prefix, IAlgorithm algorithm, IDictionary values) { var innerValues = new Dictionary(); algorithm.CollectParameterValues(innerValues); foreach (var value in innerValues) { values.Add(string.Format("{0}.{1}", prefix, value.Key), value.Value); } } protected virtual void CreateRun() { Run run = new Run(Name, Description, this); if (!SaveIndividualTrails) run.Results.Remove("Sub Algorithm Runs"); Runs.Add(run); } #region Event registry private void RegisterEvents() { if (algorithm != null) RegisterAlgorithmEvents(); if (sampleGenerationAlgorithm != null) RegisterSampleGenerationAlgorithmEvents(); } private void RegisterAlgorithmEvents() { algorithm.ExceptionOccurred += algorithm_ExceptionOccurred; algorithm.Started += algorithm_Started; algorithm.Paused += algorithm_Paused; algorithm.Prepared += algorithm_Prepared; algorithm.Stopped += algorithm_Stopped; algorithm.ExecutionTimeChanged += algorithm_ExecutionTimeChanged; algorithm.ExecutionStateChanged += algorithm_ExecutionStateChanged; algorithm.Runs.ItemsAdded += Algorithm_Runs_ItemsAdded; algorithm.ProblemChanged += algorithm_ProblemChanged; } private void DeregisterAlgorithmEvents() { algorithm.ExceptionOccurred -= algorithm_ExceptionOccurred; algorithm.Started -= algorithm_Started; algorithm.Paused -= algorithm_Paused; algorithm.Prepared -= algorithm_Prepared; algorithm.Stopped -= algorithm_Stopped; algorithm.ExecutionTimeChanged -= algorithm_ExecutionTimeChanged; algorithm.ExecutionStateChanged -= algorithm_ExecutionStateChanged; algorithm.Runs.ItemsAdded -= Algorithm_Runs_ItemsAdded; algorithm.ProblemChanged -= algorithm_ProblemChanged; } private void RegisterSampleGenerationAlgorithmEvents() { sampleGenerationAlgorithm.ExceptionOccurred += algorithm_ExceptionOccurred; sampleGenerationAlgorithm.Started += algorithm_Started; sampleGenerationAlgorithm.Paused += algorithm_Paused; sampleGenerationAlgorithm.Prepared += algorithm_Prepared; sampleGenerationAlgorithm.Stopped += sampleGenerationAlgorithm_Stopped; sampleGenerationAlgorithm.ExecutionTimeChanged += sampleGenerationAlgorithm_ExecutionTimeChanged; sampleGenerationAlgorithm.ProblemChanged += algorithm_ProblemChanged; } private void DeregisterSampleGenerationAlgorithmEvents() { sampleGenerationAlgorithm.ExceptionOccurred -= algorithm_ExceptionOccurred; sampleGenerationAlgorithm.Started -= algorithm_Started; sampleGenerationAlgorithm.Paused -= algorithm_Paused; sampleGenerationAlgorithm.Prepared -= algorithm_Prepared; sampleGenerationAlgorithm.Stopped -= sampleGenerationAlgorithm_Stopped; sampleGenerationAlgorithm.ExecutionTimeChanged -= sampleGenerationAlgorithm_ExecutionTimeChanged; sampleGenerationAlgorithm.ProblemChanged -= algorithm_ProblemChanged; } #endregion #region Sub Algorithm Event Handling void algorithm_ExecutionStateChanged(object sender, EventArgs e) { IHeuristicOptimizationProblem problem = Algorithm.Problem as IHeuristicOptimizationProblem; if (algorithm.ExecutionState == ExecutionState.Stopped && problem != null && problem.SolutionCreator is PreassignedSolutionCreator) problem.SolutionCreatorParameter.ActualValue = ((PreassignedSolutionCreator)problem.SolutionCreator).OriginalSolutionCreator; } void algorithm_ExecutionTimeChanged(object sender, EventArgs e) { ExecutionTime = executionBaseTime + algorithm.ExecutionTime; } void sampleGenerationAlgorithm_ExecutionTimeChanged(object sender, EventArgs e) { ExecutionTime = executionBaseTime + sampleGenerationAlgorithm.ExecutionTime; } void algorithm_ExceptionOccurred(object sender, EventArgs e) { OnExceptionOccurred(e.Value); ExecutionState = ExecutionState.Paused; } void algorithm_Started(object sender, EventArgs e) { running = true; ExecutionState = ExecutionState.Started; } void algorithm_Paused(object sender, EventArgs e) { ExecutionState = ExecutionState.Paused; } void algorithm_Prepared(object sender, EventArgs e) { if (executionState == ExecutionState.Stopped && !preparing) Prepare(); } void algorithm_Stopped(object sender, EventArgs e) { if (running) { executionBaseTime = executionTime + algorithm.ExecutionTime; ExecutionTime = executionBaseTime; StepNr++; PerformNextStep(); } else { ExecutionState = ExecutionState.Stopped; } } void sampleGenerationAlgorithm_Stopped(object sender, EventArgs e) { if (running) { if (sampleGenerationAlgorithm.Results.ContainsKey("Samples")) { IScope scope = sampleGenerationAlgorithm.Results["Samples"].Value as IScope; if (scope != null) PreassignedSolutionCreator.SolutionCollection = (IScope)scope.Clone(); } executionBaseTime = executionTime + sampleGenerationAlgorithm.ExecutionTime; ExecutionTime = executionBaseTime; PreassignedSolutionCreator.CurrentSolutionIndex = 0; StepNr++; PerformNextStep(); } else { ExecutionState = ExecutionState.Stopped; } } void algorithm_ProblemChanged(object sender, EventArgs e) { if (sender == algorithm && sampleGenerationAlgorithm != null) sampleGenerationAlgorithm.Problem = algorithm.Problem; if (sender == sampleGenerationAlgorithm && algorithm != null) algorithm.Problem = sampleGenerationAlgorithm.Problem; OnProblemChanged(); } void Algorithm_Runs_ItemsAdded(object sender, Collections.CollectionItemsChangedEventArgs e) { if (subAlgorithmRuns != null) foreach (var run in e.Items) { var slimRun = new Run(); foreach (var result in run.Results) slimRun.Results.Add(result); subAlgorithmRuns.Add(slimRun); slimRun.Name = string.Format("Sub Algorithm Run {0}", subAlgorithmRuns.Count); } } #endregion #region Auxiliary Functions private static void SynchronizeParameters(IParameterizedItem original, IParameterizedItem copy) { foreach (IParameter origParam in original.Parameters) { IParameter copyParam = null; if (copy.Parameters.TryGetValue(origParam.Name, out copyParam)) SynchronizeParameters(origParam, copyParam); } } private static void SynchronizeParameters(IParameter origParam, IParameter copyParam) { if (origParam == null || copyParam == null) return; IValueParameter origValueParam = origParam as IValueParameter; ILookupParameter origLookupParam = origParam as ILookupParameter; IValueParameter copyValueParam = copyParam as IValueParameter; ILookupParameter copyLookupParam = copyParam as ILookupParameter; if (origValueParam != null && copyValueParam != null && copyValueParam.DataType.IsAssignableFrom(origValueParam.DataType)) copyValueParam.Value = origValueParam.Value; if (origLookupParam != null && copyLookupParam != null) copyLookupParam.ActualName = origLookupParam.ActualName; } private void SelectPreassignedSolutionCreator() { var alg = Algorithm ?? SampleGenerationAlgorithm; if (alg == null) return; IHeuristicOptimizationProblem problem = alg.Problem as IHeuristicOptimizationProblem; if (problem != null) { if (problem.SolutionCreator is IBinaryVectorCreator && (PreassignedSolutionCreator == null || !(PreassignedSolutionCreator is IBinaryVectorCreator))) PreassignedSolutionCreator = new PreassignedBinaryVectorCreator(); else if (problem.SolutionCreator is IRealVectorCreator && (PreassignedSolutionCreator == null || !(PreassignedSolutionCreator is IRealVectorCreator))) PreassignedSolutionCreator = new PreassignedRealVectorCreator(); else if (problem.SolutionCreator is IPermutationCreator && (PreassignedSolutionCreator == null || !(PreassignedSolutionCreator is IPermutationCreator))) PreassignedSolutionCreator = new PreassignePermutationCreator(); } } #endregion } }