- Timestamp:
- 02/20/19 23:50:24 (5 years ago)
- Location:
- branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3
- Files:
-
- 7 added
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3
-
Property
svn:global-ignores
set to
tmplogs
-
Property
svn:global-ignores
set to
-
branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3/MultiTest.ps1
r16614 r16617 1 1 [cmdletbinding()] 2 2 param( 3 [int] $WaitToStep = 200,3 [int] $WaitToStep = 500, 4 4 [int] $PeelCnt = 3, 5 5 [int] $EvalCnt = 3, 6 6 [ValidateSet("method1","method2", "method3")][string] $InitMethod = "method1", 7 7 [ValidateSet("method1","method2", "method3")][string] $GrowMethod = "method1", 8 [int] $MaxJobs = 30, 9 [switch] $Clear 8 [int] $MaxJobs = 10, 9 [switch] $NoClear, 10 [switch] $UseExisting 10 11 ) 11 12 12 13 $BenchmarksPath = Resolve-Path -Relative "go-code\go-pge\data\benchmark" 13 14 $Clean = $false15 if($Clear) {16 $Clean = $true17 }18 14 19 15 $BenchFiles = Get-ChildItem -Path $BenchmarksPath -File … … 26 22 } 27 23 $ActiveJobs[$BenchName] = Start-Job -ScriptBlock { 28 param($PSPath, $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $ Clear)24 param($PSPath, $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $NoClear, $UseExisting) 29 25 Set-Location -Path $PSPath 30 ./TestResults -WaitToStep $WaitToStep -BenchName $BenchName -PeelCnt $PeelCnt -EvalCnt $EvalCnt -InitMethod $InitMethod -GrowMethod $GrowMethod *>&1 31 } -ArgumentList $(Resolve-Path $PSScriptRoot), $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $Clean 26 ./TestResults -WaitToStep $WaitToStep -BenchName $BenchName -PeelCnt $PeelCnt -EvalCnt $EvalCnt -InitMethod $InitMethod -GrowMethod $GrowMethod -NoClear:$NoClear -UseExisting:$UseExisting *>&1 27 Write-Output "Gen-Return: $LASTEXITCODE" 28 } -ArgumentList $(Resolve-Path $PSScriptRoot), $WaitToStep, $BenchName, $PeelCnt, $EvalCnt, $InitMethod, $GrowMethod, $NoClear.IsPresent, $UseExisting.IsPresent 32 29 Write-Host "Started job for $BenchName" 33 30 } … … 37 34 $Job = $ActiveJobs[$BenchName] 38 35 $Res = $Job | Receive-Job 36 $ReturnLine = $Res | Select-String -SimpleMatch "Gen-Return: " 37 if($ReturnLine -match "Gen-Return: (\d+)") { 38 if($Matches[0] -ne 0) { 39 Write-Error "$($BenchName): An error occurred! Return-Code $($Matches[0])" 40 continue 41 } 42 } 39 43 $Succeeded = ($Res | Select-String -Pattern "^[0-9]+\sOK").Count 40 44 $Failed = ($Res | Select-String -Pattern "^[0-9]+\sfailed!").Count 41 45 if($($Succeeded + $Failed) -ne $WaitToStep ) { 42 Write-Error "Something went wrong!? Please check manuelly" 46 $Missing = $WaitToStep - ($Succeeded + $Failed) 47 Write-Error "$($BenchName): $Missing steps missing, or something went wrong!? Please check manuelly" 43 48 } 44 49 if($Succeeded -eq $WaitToStep) { 45 Write-Host "$($BenchName): $Succeeded / $WaitToStep tests succeeded"50 Write-Host "$($BenchName): $Succeeded / $WaitToStep `ttests succeeded" 46 51 } else { 47 Write-Warning "$($BenchName): $Failed / $WaitToStep tests failed!"52 Write-Warning "$($BenchName): $Failed / $WaitToStep `ttests failed!" 48 53 } 49 54 } -
branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3/TestResults.ps1
r16614 r16617 1 1 [cmdletbinding()] 2 2 param( 3 [int] $WaitToStep = 200,3 [int] $WaitToStep = 30, 4 4 [string] $BenchName = "Pagie_1", 5 5 [int] $PeelCnt = 3, … … 7 7 [ValidateSet("method1","method2", "method3")][string] $InitMethod = "method1", 8 8 [ValidateSet("method1","method2", "method3")][string] $GrowMethod = "method1", 9 [bool] $Clear 9 [switch] $NoClear, 10 [switch] $UseExisting 10 11 ) 11 12 12 $GenerateNew = $true 13 14 $CLog = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "CGen_$($BenchName).log" 15 $GoLog = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "GoOut_$($BenchName).log" 16 17 $CLogError = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "CGen_$($BenchName).err" 18 $GoLogError = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "GoOut_$($BenchName).err" 13 $GenerateNew = -not $UseExisting.IsPresent 14 $MaxSecondsTimeout = 90 15 16 $LogFolder = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "tmplog" 17 18 $CLog = Join-Path -Path $LogFolder -ChildPath "CGen_$($BenchName).log" 19 $GoLog = Join-Path -Path $LogFolder -ChildPath "GoOut_$($BenchName).log" 20 21 $CLogError = Join-Path -Path $LogFolder -ChildPath "CGen_$($BenchName).err" 22 $GoLogError = Join-Path -Path $LogFolder -ChildPath "GoOut_$($BenchName).err" 19 23 20 24 $CExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'testmulti.exe' 21 25 $GoExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'go-code\go-pge\go-pge.exe' 22 26 27 $BenchData = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'go-code\go-pge\data\benchmark' 28 $TestDataFile = Join-Path -Path $BenchData -ChildPath "$($BenchName).trn" 29 $TrainDataFile = Join-Path -Path $BenchData -ChildPath "$($BenchName).tst" 30 31 $ConfigData = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'go-code\go-pge\config\prob\bench' 32 $ConfigDataFile = Join-Path -Path $ConfigData -ChildPath "$($BenchName).cfg" 33 34 if(-not $(Test-Path -Path $TestDataFile -PathType Leaf) -or -not $(Test-Path -Path $TrainDataFile -PathType Leaf)) { 35 Write-Error "Train/test file does't exist for benchmark: $TestDataFile" 36 exit 4 37 } 38 if(-not $(Test-Path -Path $ConfigDataFile -PathType Leaf)) { 39 Write-Error "Configfile doesn't exist for benchmark: $ConfigDataFile" 40 exit 4 41 } 42 43 if($NoClear.IsPresent) { 44 $Clear = $false 45 } else { 46 $Clear = $GenerateNew 47 } 48 23 49 if($GenerateNew) { 50 if(-not $(Test-Path $LogFolder -Type Container)) { 51 New-Item -Path $LogFolder -ItemType Directory -Force 52 } 53 24 54 if(Test-Path $GoLog -Type Leaf) { 25 55 Remove-Item -Path $GoLog … … 37 67 $CGenProc = Start-Process -FilePath $CExecutable -ArgumentList "$BenchName -evals=$($EvalCnt) -peel=$($PeelCnt) -iter=$($WaitToStep) -init=$($InitMethod) -grow=$($GrowMethod)" -RedirectStandardOutput $CLog -RedirectStandardError $CLogError -WorkingDirectory $PSScriptRoot -PassThru 38 68 $StepReached = $false 69 $LastLogLength = 0 70 $Timeout = $MaxSecondsTimeout 39 71 while(-not $StepReached) { 40 72 $CGenCont = Get-Content $CLog … … 42 74 $StepReached = $true 43 75 } 76 if($LastLogLength -eq $CGenCont.Length) { 77 $Timeout-- 78 } else { 79 $Timeout = $MaxSecondsTimeout 80 } 81 if($Timeout -le 0) { 82 $CGenProc | Stop-Process -ErrorAction SilentlyContinue -Force 83 Write-Error "C-data generation timed out!" 84 exit 2 85 } 44 86 Start-Sleep -Seconds 1 87 $LastLogLength = $CGenCont.Length 45 88 } 46 89 … … 52 95 $GoGenProc = Start-Process -FilePath $GoExecutable -ArgumentList "-pcfg=prob/bench/$($BenchName).cfg -evals=$($EvalCnt) -peel=$($PeelCnt) -iter=$($WaitToStep) -init=$($InitMethod) -grow=$($GrowMethod)" -RedirectStandardOutput $GoLog -RedirectStandardError $GoLogError -WorkingDirectory $(Split-Path -Path $GoExecutable -Parent) -PassThru 53 96 $StepReached = $false 97 $Timeout = $MaxSecondsTimeout 98 $LastLogLength = 0 54 99 while(-not $StepReached) { 55 100 $GoGenCont = Get-Content $GoLog … … 57 102 $StepReached = $true 58 103 } 104 if($LastLogLength -eq $CGenCont.Length) { 105 $Timeout-- 106 } else { 107 $Timeout = $MaxSecondsTimeout 108 } 109 if($Timeout -le 0) { 110 $GoGenProc | Stop-Process -ErrorAction SilentlyContinue -Force 111 Write-Error "Go-data generation timed out!" 112 exit 3 113 } 59 114 Start-Sleep -Seconds 1 115 $LastLogLength = $CGenCont.Length 60 116 } 61 117 $GoGenProc | Stop-Process -ErrorAction SilentlyContinue … … 122 178 $ParseGo = Get-GoSteps -Content $GoContent 123 179 124 for each($StepNr in $($ParseGo.Keys | Sort-Object | Where-Object { $_ -ne 0 -and $_ -le $WaitToStep})) {180 for($StepNr = 1; $StepNr -le $WaitToStep; $StepNr++) { 125 181 $GoResult = $ParseGo[$StepNr] 126 182 [System.Collections.ArrayList]$CResult = $ParseC[$StepNr] 127 128 if($null -eq $CResult -or $CResult.Count -lt 1) { 129 Write-Warning "Missing CStep nr. $StepNr" 130 continue 131 } 132 if($null -eq $GoResult -or $GoResult.Count -lt 1) { 133 Write-Warning "Missing GoStep nr. $StepNr" 134 continue 135 } 136 137 $StepResult = $true 138 foreach($GoRes in $GoResult) { # Step Results 139 $State = $false 140 foreach($CRes in $CResult.Clone()) { # Compare every Result with every Step Result 141 Write-Output "$($GoRes) with $($CRes)" 142 if(Compare-Results -First $GoRes -Second $CRes) { 143 $State = $true 144 $CResult.Remove($CRes) 145 break 183 if($null -eq $GoResult -or $null -eq $CResult) { 184 $StepResult = $false 185 } else { 186 $StepResult = $CResult.Count -eq $GoResult.Count 187 } 188 if($StepResult) { 189 foreach($GoRes in $GoResult) { # Step Results 190 $State = $false 191 foreach($CRes in $CResult.Clone()) { # Compare every Result with every Step Result 192 Write-Debug "$($GoRes) with $($CRes)" 193 if(Compare-Results -First $GoRes -Second $CRes) { 194 $State = $true 195 $CResult.Remove($CRes) 196 break 197 } 146 198 } 147 }148 if(-not $State) {149 $StepResult = $false199 if(-not $State) { 200 $StepResult = $false 201 } 150 202 } 151 203 }
Note: See TracChangeset
for help on using the changeset viewer.