source: branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3/MultiTest.ps1 @ 16615

[cmdletbinding()]
param(
    [int] $WaitToStep = 200,
    [int] $PeelCnt = 3,
    [int] $EvalCnt = 3,
    [ValidateSet("method1","method2", "method3")][string] $InitMethod = "method1",
    [ValidateSet("method1","method2", "method3")][string] $GrowMethod = "method1",
    [int] $MaxJobs = 30,
    [switch] $Clear
)

$BenchmarksPath = Resolve-Path -Relative "go-code\go-pge\data\benchmark"

$Clean = $false
if($Clear) {
    $Clean = $true
}

$BenchFiles = Get-ChildItem -Path $BenchmarksPath -File
$ActiveJobs = @{}
foreach($BenchName in $($BenchFiles | Select-Object -ExpandProperty BaseName -Unique)) {
    $RunningCnt = ($ActiveJobs.Values | Where-Object { $_.State -eq "Running" }).Count
    while ($RunningCnt -gt $MaxJobs) {
        Start-Sleep -Seconds 1
        $RunningCnt = ($ActiveJobs.Values | Where-Object { $_.State -eq "Running" }).Count
    }
    $ActiveJobs[$BenchName] = Start-Job -ScriptBlock {
        param($PSPath, $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $Clear)
        Set-Location -Path $PSPath
        ./TestResults -WaitToStep $WaitToStep -BenchName $BenchName -PeelCnt $PeelCnt -EvalCnt $EvalCnt -InitMethod $InitMethod -GrowMethod $GrowMethod *>&1
    } -ArgumentList $(Resolve-Path $PSScriptRoot), $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $Clean
    Write-Host "Started job for $BenchName"
}

$ActiveJobs.Values | Wait-Job | Out-Null
foreach($BenchName in $ActiveJobs.Keys) {
    $Job = $ActiveJobs[$BenchName]
    $Res = $Job | Receive-Job
    $Succeeded = ($Res | Select-String -Pattern "^[0-9]+\sOK").Count
    $Failed = ($Res | Select-String -Pattern "^[0-9]+\sfailed!").Count
    if($($Succeeded + $Failed) -ne $WaitToStep ) {
        Write-Error "Something went wrong!? Please check manuelly"
    }
    if($Succeeded -eq $WaitToStep) {
        Write-Host "$($BenchName): $Succeeded / $WaitToStep tests succeeded"
    } else {
        Write-Warning "$($BenchName): $Failed / $WaitToStep tests failed!"
    }
}
Get-Job | Remove-Job -Force | Out-Null