source: branches/2929_PrioritizedGrammarEnumeration/HeuristicLab.Algorithms.DataAnalysis.PGE/3.3/TestResults.ps1 @ 17578

[cmdletbinding()]
param(
    [int] $WaitToStep = 200,
    [parameter(Position=0)][string] $BenchName = "Pagie_1",
    [int] $PeelCnt = 3,
    [int] $EvalCnt = 3,
    [ValidateSet("method1","method2", "method3")][string] $InitMethod = "method1",
    [ValidateSet("method1","method2", "method3")][string] $GrowMethod = "method1",
    [switch] $NoClear,
    [switch] $UseExisting,
    [switch] $i386,
    [int]$MaxSecondsTimeout = 6000
)

$GenerateNew = -not $UseExisting.IsPresent

$LogFolder = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath "tmplog"

$CLog = Join-Path -Path $LogFolder -ChildPath "$($BenchName)_CGen.log"
$GoLog = Join-Path -Path $LogFolder -ChildPath "$($BenchName)_GoOut.log"

$CLogError = Join-Path -Path $LogFolder -ChildPath "$($BenchName)_CGen.err"
$GoLogError = Join-Path -Path $LogFolder -ChildPath "$($BenchName)_GoOut.err"

if($i386.IsPresent) {
    $CExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\testmulti32.exe'
    $GoExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\go-pge32.exe'
} else {
    $CExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\testmulti.exe'
    $GoExecutable = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\go-pge.exe'
}

$BenchData = Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\data\benchmark'
$TestDataFile = Join-Path -Path $BenchData -ChildPath "$($BenchName).trn"
$TrainDataFile = Join-Path -Path $BenchData -ChildPath "$($BenchName).tst"

$ConfigData =  Join-Path -Path $(Resolve-Path $PSScriptRoot) -ChildPath 'src\go-pge\config\prob\bench'
$ConfigDataFile = Join-Path -Path $ConfigData -ChildPath "$($BenchName).cfg"

if(-not $(Test-Path -Path $TestDataFile -PathType Leaf) -or -not $(Test-Path -Path $TrainDataFile -PathType Leaf)) {
    Write-Error "Train/test file does't exist for benchmark: $TestDataFile"
    exit 4
}
if(-not $(Test-Path -Path $ConfigDataFile -PathType Leaf)) {
    Write-Error "Configfile doesn't exist for benchmark: $ConfigDataFile"
    exit 4
}

if($NoClear.IsPresent) {
    $Clear = $false
} else {
    $Clear = $GenerateNew   
}

$env:GOGC="off"
$env:GODEBUG="cgocheck=0"
$env:CGO_ENABLED=1

if(-not $(Test-Path $LogFolder -Type Container)) {
   New-Item -Path $LogFolder -ItemType Directory -Force
}

if($GenerateNew -or -not $(Test-Path -Path $CLog)) {
  if(Test-Path $CLog -Type Leaf) {
    Remove-Item -Path $CLog
    }
   if(Test-Path $CLogError -Type Leaf) {
      Remove-Item -Path $CLogError
   }
    $CGenProc = Start-Process -FilePath $CExecutable -ArgumentList "$BenchName -evals=$($EvalCnt) -peel=$($PeelCnt) -iter=$($WaitToStep) -init=$($InitMethod) -grow=$($GrowMethod)" -RedirectStandardOutput $CLog -RedirectStandardError $CLogError -WorkingDirectory $(Split-Path -Path $GoExecutable -Parent) -PassThru
    $StepReached = $false
    $LastLogLength = 0
    $Timeout = $MaxSecondsTimeout
    while(-not $StepReached) {
        $CGenCont = Get-Content $CLog
        if($CGenCont | Select-String -Pattern "\*+ StepW: $WaitToStep \*+") {
            $StepReached = $true
        }
        if(-not $StepReached -and $LastLogLength -eq $CGenCont.Length) {
            $Timeout--
        } else {
            $Timeout = $MaxSecondsTimeout
        }
        if($Timeout -le 0) {
            $CGenProc | Stop-Process -ErrorAction SilentlyContinue -Force
            Write-Error "C-data generation timed out!"
            exit 2
        }
        $LastLogLength = $CGenCont.Length
        Start-Sleep -Seconds 1
    }
    $CGenProc  | Stop-Process -ErrorAction SilentlyContinue
}
 
if($GenerateNew -or -not $(Test-Path -Path $GoLog)) { 
   if(Test-Path $GoLogError -Type Leaf) {
      Remove-Item -Path $GoLogError
   }
   if(Test-Path $GoLog -Type Leaf) {
    Remove-Item -Path $GoLog
  }
    $GoGenProc = Start-Process -FilePath $GoExecutable -ArgumentList "-pcfg=prob/bench/$($BenchName).cfg -evals=$($EvalCnt) -peel=$($PeelCnt) -iter=$($WaitToStep) -init=$($InitMethod) -grow=$($GrowMethod)"  -RedirectStandardOutput $GoLog -RedirectStandardError $GoLogError -WorkingDirectory $(Split-Path -Path $GoExecutable -Parent) -PassThru
    $StepReached = $false
    $Timeout = $MaxSecondsTimeout
    $LastLogLength = 0
    while(-not $StepReached) {
        $GoGenCont = Get-Content $GoLog
        if($GoGenCont | Select-String -Pattern "PS\.step\(\)\s+$($WaitToStep)") {
            $StepReached = $true
        }
        if($LastLogLength -eq $CGenCont.Length) {
            $Timeout--
        } else {
            $Timeout = $MaxSecondsTimeout
        }
        if(-not $StepReached -and $Timeout -le 0) {
            $GoGenProc | Stop-Process -ErrorAction SilentlyContinue -Force
            Write-Error "Go-data generation timed out!"
            exit 3
        }
        $LastLogLength = $CGenCont.Length
        Start-Sleep -Seconds 1
    }
    $GoGenProc | Stop-Process -ErrorAction SilentlyContinue
}

# Read Result

function Get-CSteps {
    param([Object[]] $Content)
    $ValueList = @{}
    $CurrentStep = 0
    foreach($Line in $Content) {
        if($Line -match "\*+ StepW: (\d+) \*+") {
            [int]$CurrentStep = $Matches[1]
            $ValueList[$CurrentStep] = $(New-Object -TypeName System.Collections.ArrayList)
        } elseif($Line -match "push\/pop \(\d+\,\d+\) (.+)") {
            $ValueList[$CurrentStep].Add($Matches[1].Trim().Replace('(', '').Replace(')', '').Replace(' ', '')) | Out-Null
        }
    }
    return $ValueList
}

function Get-GoSteps {
    param([Object[]] $Content)
    $ValueList = @{}
    $CurrentStep = 0
    foreach($Line in $Content) {
        if($Line -match "PS\.step\(\)\s+(\d+)") {
            [int]$CurrentStep = $Matches[1]
            $ValueList[$CurrentStep] = $(New-Object -TypeName System.Collections.ArrayList)
        } elseif($Line -match "pop\/push\(\d+\,\d+\)\: (.+)") {
            $ValueList[$CurrentStep].Add($Matches[1].Trim().Replace('(', '').Replace(')', '').Replace(' ', '')) | Out-Null
        }
    }
    return $ValueList
}

function Compare-Results {
    param(
        [string]$First,
        [string]$Second
    )
    $FirstChars = $First.Trim().ToUpper().ToCharArray() | Sort-Object
    $SecondChars = $Second.Trim().ToUpper().ToCharArray() | Sort-Object

    [System.Collections.ArrayList]$LeftChars = $SecondChars.Clone()
    $FirstCharNr = 0
    foreach($FirstChar in $FirstChars) {
        $SecondChar = $SecondChars[$FirstCharNr]
        if($FirstChar -eq $SecondChar) {
            $LeftChars.Remove($FirstChar)
        } else {
            return $false
        }
        $FirstCharNr++
    }
    return [bool]($LeftChars.Count -eq 0)
}

$CContent = Get-Content -Path $CLog
$GoContent = Get-Content -Path $GoLog

$ParseC = Get-CSteps -Content $CContent
$ParseGo = Get-GoSteps -Content $GoContent

for($StepNr = 1; $StepNr -le $WaitToStep; $StepNr++) {
    $GoResult = $ParseGo[$StepNr]
    [System.Collections.ArrayList]$CResult = $ParseC[$StepNr]
    if($null -eq $GoResult -or $null -eq $CResult) {
        $StepResult = $false
    } else {
        $StepResult = $CResult.Count -eq $GoResult.Count
    }
    if($StepResult) {    
        foreach($GoRes in $GoResult) {            # Step Results
            $State = $false
            foreach($CRes in $CResult.Clone()) {  # Compare every Result with every Step Result
                Write-Debug "$($GoRes) with $($CRes)"
                if(Compare-Results -First $GoRes -Second $CRes) {
                    $State = $true
                    $CResult.Remove($CRes)
                    break
                }
            }
            if(-not $State) {
                $StepResult = $false
            }
        }
    }

    if($StepResult) {
        Write-Output "$StepNr OK"
    } else {
        Write-Warning "$StepNr failed!"
    }
}

if($Clear) {
    if(Test-Path $GoLog -Type Leaf) {
        Remove-Item -Path $GoLog
    }
    if(Test-Path $CLog -Type Leaf) {
        Remove-Item -Path $CLog
    }
    if(Test-Path $GoLogError -Type Leaf) {
        Remove-Item -Path $GoLogError
    }
    if(Test-Path $CLogError -Type Leaf) {
        Remove-Item -Path $CLogError
    }
}