# maintainProcess.ps1
# version 0.0.1
$computernames=@(
'server1',
'server2'
)
$processName='cmd'
$processPath='C:\WINDOWS\system32\cmd.exe'
$minutesToDefineCrashed=1 # this marker is only valid if the process is marked as unresponsive by the system
$runMinutes=1
$credentials=$null
$maxMinutesPerJob=10
$verbose=$true
function maintainProcess{
param(
$targetMachines=$env:computername,
$processname='cmd',
$processPath='C:\Windows\system32\cmd.exe',
$minutesToDefineCrashed=1,
$runMinutes=2,
$credentials
)
$actionOnCrash={param($processName,$processPath)
stop-process -name $processName -force -ea Ignore
if($processPath){$null=start-process -FilePath $processPath}else{$null=start-process -Name $processName}
}
function restartProcess($actionOnCrash,$processname,$processPath,$psSession){
invoke-command -session $psSession -scriptblock $actionOnCrash -Arg $processname,$processPath
$processIsRunning=$null -ne (invoke-command -session $psSession -scriptblock {param($processname)get-process -name $processName -EA SilentlyContinue} -Args $processname)
if($processIsRunning){
write-host "$processName has successfully restarted on $targetMachine"
return $true
}else{
write-host "$processName has NOT successfully restarted on $targetMachine"
return $false
}
}
$results=[hashtable]@{}
foreach($targetMachine in $targetMachines){
$overallClock=[System.Diagnostics.Stopwatch]::StartNew()
$psSession=if($credentials){
New-PSSession $targetMachine -Credential $credentials
}else{
New-PSSession $targetMachine
}
if($psSession.State -eq 'Opened'){
$previousCpuConsumption=0
write-host "Checking CPU consumption of $processName on $targetMachine"
do{
$iterationClock=[System.Diagnostics.Stopwatch]::StartNew()
$process=Invoke-Command -Session $psSession {param($processname) Get-Process $processname -EA SilentlyContinue} -Args $processName
if($null -eq $process){
write-warning "$processname is NOT running on $targetMachine"
$result=restartProcess $actionOnCrash $processname $processPath $psSession
}else{
$responding=$process.Responding
$currentCpuConsumption=$process.CPU
write-host $currentCpuConsumption
$cpuConsumptionChanged=$currentCpuConsumption -ne $previousCpuConsumption
$noActivities=$iterationClock.elapsed.totalminutes -ge $minutesToDefineCrashed
if($cpuConsumptionChanged){
$null=$iterationClock.reset
$previousCpuConsumption=$currentCpuConsumption
$result=$true
}elseif(!$responding -and $noActivities){
write-warning "$processName has CRASHED on $targetMachine as defined its RESPONDING flag equal False and there are no activities."
$result=restartProcess $actionOnCrash $processname $processPath $psSession
}
sleep -Seconds 10
}
} until ($overallClock.elapsed.totalminutes -ge $runMinutes)
Remove-PSSession $psSession
$minutesElapsed=[math]::round($overallClock.Elapsed.TotalMinutes,2)
write-host "Runtime of $minutesElapsed minutes has elapsed for $targetMachine"
$results+=[hashtable]@{$targetMachine=$result}
}else{
write-warning "Unable to open a WinRM session to $targetMachine.`r`nPlease monitor it's progress manually."
$results+=[hashtable]@{$targetMachine=$null}
}
}
return $results
}
# maintainProcess $targetMachines $processname $processPath $minutesToDefineCrashed $runMinutes $credential
function maintainProcessParallel{
param(
$computerNames=$env:computername,
$processname='cmd',
$processPath='C:\Windows\system32\cmd.exe',
$minutesToDefineCrashed=1, # this marker is only valid if the process is marked as unresponsive by the system
$runMinutes=2,
$credentials,
$maxMinutesPerJob=10,
$verbose=$true
)
$timer=[System.Diagnostics.Stopwatch]::StartNew()
$jobResults=@()
$lineBreak=60
$dotCount=0
$minute=0
$processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
$cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
$maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
$jobtimer = @{}
foreach($computerName in $computerNames){
$thisIterationCompleted=$false
do {
$jobsCount=(Get-Job -State 'Running').Count
if ($jobsCount -lt $maxSimultaneousJobs){
if($verbose){write-host "Initiating job for $computerName"}
$job=Start-Job -name $computerName -ScriptBlock {
param($maintainProcess,$computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credentials)
[scriptblock]::create($maintainProcess).invoke($computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credential)
} -Args ${function:maintainProcess},$computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credentials
$jobtimer[$job.Id]=[System.Diagnostics.Stopwatch]::startnew()
$thisIterationCompleted=$true
}else{
if($verbose){
if($dotCount++ -lt $lineBreak){
write-host '.' -NoNewline
}else{
$minute++
write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
$dotCount=0
}
}
sleep -seconds 1
}
$expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
if($expiredJobs){
$expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
$expiredJobs.Name|%{$jobTimer.Remove($_)}
}
}until ($thisIterationCompleted)
}
$totalJobsCount=(get-job).count
$processedCount=0
while($processedCount -lt $totalJobsCount){
$completedJobs=get-job|?{$_.State -eq 'Completed'}
$stoppedJobs=get-job|?{$_.State -eq 'Stopped'}
$expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
if($expiredJobs){
$expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
$expiredJobs.Name|%{$jobTimer.Remove($_)}
}
if($completedJobs){
foreach ($job in $completedJobs){
$computer=$job.Name
if($verbose){
write-host "`r`n===================================================`r`n$computer job COMPLETED with these messages:`r`n===================================================`r`n"
}
$jobResult=receive-job -id $job.id
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
if($stoppedJobs){
foreach ($job in $stoppedJobs){
$computer=$job.Name
if($verbose){
write-host "`r`n===================================================`r`n$computer job STOPPED with these messages:`r`n===================================================`r`n" -ForegroundColor Red
}
$jobResult=receive-job -id $job.id
# $jobResults+=,$jobResult
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$jobResult=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computer
stoppedServices='serverTimeout'
fixedAutomatically=$False
}
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
# Safeguard against stuck jobs
if($timer.elapsed.totalminutes -ge $maxMinutesPerJob){
get-job|Remove-Job -Force
write-warning "There were some errors in this iteration. Shell was aborted to mitigate potential persistency issues."
exit
}
}
$minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
$timer.stop()
write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
return $jobResults #|select -property * -excludeproperty RunspaceId
}
maintainProcessParallel $computerNames `
$processname `
$processPath `
$minutesToDefineCrashed `
$runMinutes `
$credentials `
$maxMinutesPerJob `
$verbose
Categories: