source: branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3/MultiTest.ps1

[cmdletbinding()]
param(
    [int] $WaitToStep = 200,
    [int] $PeelCnt = 3,
    [int] $EvalCnt = 3,
    [ValidateSet("method1","method2", "method3")][string] $InitMethod = "method1",
    [ValidateSet("method1","method2", "method3")][string] $GrowMethod = "method1",
    [int] $MaxJobs = 3,
    [switch] $NoClear,
    [switch] $UseExisting
)

$BenchmarksPath = Resolve-Path -Relative "src\go-pge\data\benchmark"

$env:GOGC="off"
$env:GODEBUG="cgocheck=0"
$env:CGO_ENABLED=1

$BenchFiles = Get-ChildItem -Path $BenchmarksPath -File
$ActiveJobs = @{}
foreach($BenchName in $($BenchFiles | Select-Object -ExpandProperty BaseName -Unique)) {
    $RunningCnt = ($ActiveJobs.Values | Where-Object { $_.State -eq "Running" }).Count
    while ($RunningCnt -gt $MaxJobs) {
        Start-Sleep -Seconds 1
        $RunningCnt = ($ActiveJobs.Values | Where-Object { $_.State -eq "Running" }).Count
    }
    $ActiveJobs[$BenchName] = Start-Job -ScriptBlock {
        param($PSPath, $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $NoClear, $UseExisting)
        Set-Location -Path $PSPath
        ./TestResults -WaitToStep $WaitToStep -BenchName $BenchName -PeelCnt $PeelCnt -EvalCnt $EvalCnt -InitMethod $InitMethod -GrowMethod $GrowMethod -NoClear:$NoClear -UseExisting:$UseExisting *>&1
        Write-Output "Gen-Return: $LASTEXITCODE"
    } -ArgumentList $(Resolve-Path $PSScriptRoot), $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $NoClear.IsPresent, $UseExisting.IsPresent
    Write-Host "Started job for $BenchName"
}

$ActiveJobs.Values | Wait-Job | Out-Null
foreach($BenchName in $ActiveJobs.Keys | Sort-Object) {
    $Job = $ActiveJobs[$BenchName]
    $Res = $Job | Receive-Job
    $ReturnLine = $Res | Select-String -SimpleMatch "Gen-Return: "
    if($ReturnLine -match "Gen-Return: (\d+)") {
        if($Matches[0] -ne 0) {
            Write-Error "$($BenchName): An error occurred! Return-Code $($Matches[0])"
            continue
        }
    }
    $Succeeded = ($Res | Select-String -Pattern "^[0-9]+\sOK").Count
    $Failed = ($Res | Select-String -Pattern "^[0-9]+\sfailed!").Count
    if($($Succeeded + $Failed) -ne $WaitToStep ) {
        $Missing = $WaitToStep - ($Succeeded + $Failed)
        Write-Error "$($BenchName): $Missing steps missing, or something went wrong!? Please check manuelly"
    }
    if($Succeeded -eq $WaitToStep) {
        Write-Host "$($BenchName): $Succeeded / $WaitToStep `ttests succeeded"
    } else {
        Write-Warning "$($BenchName): $Failed / $WaitToStep `ttests failed!"
    }
}
Get-Job | Remove-Job -Force | Out-Null