PowerShell: Maintain Windows Services Remotely via WinRM

# maintainServices.ps1

# FQDN of each computer name (required)
$computerList=@'
testWindows.intranet.kimconnect.com
testWindows.dmz.dragoncoin.com
'@


$serviceNames=@(
    'windows_exporter',
    'nscp',
    'rpcss'
)

$keychain=[hashtable]@{
    'intranet.kimconnect.com'=@{username='intranet\svc-maintenance';password=$env:password}
    'dmz.dragoncoin.com'=@{username='dmz\svc-maintenance';password=$env:password}
}

function maintainService($serviceName='Spooler',$desiredState='Running'){
    $statesDictionary=[hashtable]@{
        'Running'='Start-Service'
        'Stopped'='Stop-Service -Force'
        'Starting'='Start-Sleep -Seconds 30;Start-Service'
        'Stopping'='Start-Sleep -Seconds 30;Stop-Service -Force'
    }
    $service=try{get-service -Name $serviceName}catch{$false}
    if($service.State -ne $desiredState){

    }
}

function simultaneousExec{
    param(
        $computerNames,
        $credentials,
        $functionName,
        $arguments,
        $maxMinutesPerJob=2
    )
    $timer=[System.Diagnostics.Stopwatch]::StartNew()
    $jobResults=@()
    $lineBreak=60
    $dotCount=0
    $minute=0
    $processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
    $cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
    $maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
    write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
    $jobtimer = @{}
    foreach($computerName in $computerNames){
        $thisIterationCompleted=$false
        do {
            $jobsCount=(Get-Job -State 'Running').Count
            if ($jobsCount -lt $maxSimultaneousJobs){            
                if($verbose){write-host "Initiating job for $computerName"}
                $job=Start-Job -name $computerName -ScriptBlock {
                    param($functionName,$computerName,$credentials,$arguments)
                    [scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$arguments)
                } -Args ${function:functionName},$computerName,$credentials,$arguments
                $jobtimer[$job.Id] = [System.Diagnostics.Stopwatch]::startnew()
                $thisIterationCompleted=$true
            }else{
                if($verbose){
                    if($dotCount++ -lt $lineBreak){
                        write-host '.' -NoNewline
                    }else{
                        $minute++
                        write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
                        $dotCount=0
                        }
                }
                sleep -seconds 1
            }
            $expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
            if($expiredJobs){
                $expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
                $expiredJobs.Name|%{$jobTimer.Remove($_)}
            }
        }until($thisIterationCompleted)
    }
    $totalJobsCount=(get-job).count
    $processedCount=0
    while($processedCount -lt $totalJobsCount){
        $completedJobs=get-job|?{$_.State -eq 'Completed'}
        $stoppedJobs=get-job|?{$_.State -eq 'Stopped'}
        $expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
        if($expiredJobs){
            $expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
            $expiredJobs.Name|%{$jobTimer.Remove($_)}
        }
        if($completedJobs){
            foreach ($job in $completedJobs){
                $computer=$job.Name
                if($verbose){
                    write-host "`r`n===================================================`r`n$computer job COMPLETED with these messages:`r`n===================================================`r`n"
                }
                $jobResult=receive-job -id $job.id
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }
        if($stoppedJobs){
            foreach ($job in $stoppedJobs){
                $computer=$job.Name
                if($verbose){
                    write-host "`r`n===================================================`r`n$computer job STOPPED with these messages:`r`n===================================================`r`n" -ForegroundColor Red
                }
                $jobResult=receive-job -id $job.id
                # $jobResults+=,$jobResult
                $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
                $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
                $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
                $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
                $jobResult=[pscustomobject]@{
                    timeStamp=$timeStamp
                    computerName=$computer
                    stoppedServices='Unknown'
                    fixedAutomatically=$False
                    }
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }
    
        # Safeguard against stuck jobs
        if($timer.elapsed.totalminutes -ge $maxMinutesPerJob){
            $expiredJobs=get-job
            if($expiredJobs.count -gt 0){
                $expiredJobNames=$expiredJobs.Name
                $expiredJobs|Remove-Job -Force
                write-warning "Shell was aborted to mitigate potential persistency issues due to these expired jobs:`r`n$expiredJobNames"                
            }else{
                write-host "The preset $maxMinutesPerJob minutes have expired. Exiting function."
                $null=get-job|Remove-Job -Force
            }
            break
        }    
    }
    $minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
    $timer.stop()
    write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
    return $jobResults #|select -property * -excludeproperty RunspaceId
}

function forceRestartService($serviceName='Spooler'){    
    $processId=Get-WmiObject -Class Win32_Service -Filter "Name LIKE '$serviceName'"|Select-Object -ExpandProperty ProcessId
    if($processId.count -eq 1 -and $processId -ne 0){
        try{
            $taskkill=taskkill /f /pid $processid
            write-host $taskkill
            start-service $serviceName
            $result=get-service $serviceName
            if($result.Status -eq 'Running'){
                $processId=Get-WmiObject -Class Win32_Service -Filter "Name LIKE '$serviceName'"|Select-Object -ExpandProperty ProcessId
                write-host "$serviceName has successfully restarted with pid $processId" -foregroundcolor GREEN
                return $true
            }else{
                write-host "$serviceName has NOT successfully restarted" -foregroundcolor RED
                return $false
            }
        }catch{
            write-warning $_
            return $false
        }
    }else{
        try{
            start-service $serviceName
        }catch{
            write-warning $_
            # write-warning "$serviceName is not found on $env:computername"
            return $null
        }
        
    }
}


# Convert text to array
$computernames=@($computerList -split "`n")|%{$_.Trim()}
# Convert credentials from keychain automatically
$credentials=[hashtable]@{}
$keychain.GetEnumerator()|%{$credentials+=[hashtable]@{
    $_.Name=New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $_.value.username,$(ConvertTo-SecureString $_.value.Password -AsPlainText -Force)
    }}
# Map computernames to credentials
$computerCreds=[hashtable]@{}
foreach($computername in $computernames){
    $domain=[regex]::match($computername,'(\.)(.*)').groups[2].value
    $credential=$credentials[$domain]
    $computerCreds+=[hashtable]@{$computername=$credential}
}
$results=[hashtable]@{}
foreach($computer in $computerCreds.GetEnumerator()){    
    $pssession=try{new-pssession -computername $computer.name -credential $computer.value}catch{$null}
    if($pssession.state -eq 'Opened'){
        write-host "Connected to $($computer.name) using $($computer.value.username)"
        $result=invoke-command -session $pssession {
            param($serviceNames,$forceRestartService)
            $services=get-service $serviceNames -EA SilentlyContinue
            if($services.Status|?{'Running' -ne $_}){
                write-warning "$($services.Name) is/are NOT running on $env:computername"
                try{
                    foreach($service in $services){
                        if($service.Status -eq 'Starting'){
                            [scriptblock]::create($forceRestartService).invoke($service.name)
                        }else{
                            start-service $service.Name
                        }
                    }
                    # start-service $services.Name
                    return $true
                }catch{
                    write-warning $_
                    return $false
                }
            }else{
                write-host "$($services.Name) is/are running on $env:computername" -foregroundcolor Green
                return $true
            }
        } -Args (,$serviceNames)#,${function:forceRestartService}
        remove-pssession $pssession
        $results+=[hashtable]@{$computer.Name=if($result){'Passed'}else{'Failed'}}
    }else{
        write-warning "Unable to connect to $($computer.Name)"
        $results+=[hashtable]@{$computer.Name='Unknown'}
    }    
}

$passed=$results.GetEnumerator()|?{$_.Value -eq 'Passed'}
$failed=$results.GetEnumerator()|?{$_.Value -eq 'Failed'}
$unknown=$results.GetEnumerator()|?{$_.Value -eq 'Unknown'}

if($null -ne $passed){
    write-host "Passed:`r`n"
    $passed|out-string|write-host
}
if($null -ne $failed){
    write-host "Failed:`r`n"
    $failed|out-string|write-host
}
if($null -ne $unknown){
    write-host "Unknowns:`r`n"
    $unknown|out-string|write-host
}