# maintainServices.ps1
# FQDN of each computer name (required)
$computerList=@'
testWindows.intranet.kimconnect.com
testWindows.dmz.dragoncoin.com
'@
$serviceNames=@(
'windows_exporter',
'nscp',
'rpcss'
)
$keychain=[hashtable]@{
'intranet.kimconnect.com'=@{username='intranet\svc-maintenance';password=$env:password}
'dmz.dragoncoin.com'=@{username='dmz\svc-maintenance';password=$env:password}
}
function maintainService($serviceName='Spooler',$desiredState='Running'){
$statesDictionary=[hashtable]@{
'Running'='Start-Service'
'Stopped'='Stop-Service -Force'
'Starting'='Start-Sleep -Seconds 30;Start-Service'
'Stopping'='Start-Sleep -Seconds 30;Stop-Service -Force'
}
$service=try{get-service -Name $serviceName}catch{$false}
if($service.State -ne $desiredState){
}
}
function simultaneousExec{
param(
$computerNames,
$credentials,
$functionName,
$arguments,
$maxMinutesPerJob=2
)
$timer=[System.Diagnostics.Stopwatch]::StartNew()
$jobResults=@()
$lineBreak=60
$dotCount=0
$minute=0
$processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
$cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
$maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
$jobtimer = @{}
foreach($computerName in $computerNames){
$thisIterationCompleted=$false
do {
$jobsCount=(Get-Job -State 'Running').Count
if ($jobsCount -lt $maxSimultaneousJobs){
if($verbose){write-host "Initiating job for $computerName"}
$job=Start-Job -name $computerName -ScriptBlock {
param($functionName,$computerName,$credentials,$arguments)
[scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$arguments)
} -Args ${function:functionName},$computerName,$credentials,$arguments
$jobtimer[$job.Id] = [System.Diagnostics.Stopwatch]::startnew()
$thisIterationCompleted=$true
}else{
if($verbose){
if($dotCount++ -lt $lineBreak){
write-host '.' -NoNewline
}else{
$minute++
write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
$dotCount=0
}
}
sleep -seconds 1
}
$expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
if($expiredJobs){
$expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
$expiredJobs.Name|%{$jobTimer.Remove($_)}
}
}until($thisIterationCompleted)
}
$totalJobsCount=(get-job).count
$processedCount=0
while($processedCount -lt $totalJobsCount){
$completedJobs=get-job|?{$_.State -eq 'Completed'}
$stoppedJobs=get-job|?{$_.State -eq 'Stopped'}
$expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
if($expiredJobs){
$expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
$expiredJobs.Name|%{$jobTimer.Remove($_)}
}
if($completedJobs){
foreach ($job in $completedJobs){
$computer=$job.Name
if($verbose){
write-host "`r`n===================================================`r`n$computer job COMPLETED with these messages:`r`n===================================================`r`n"
}
$jobResult=receive-job -id $job.id
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
if($stoppedJobs){
foreach ($job in $stoppedJobs){
$computer=$job.Name
if($verbose){
write-host "`r`n===================================================`r`n$computer job STOPPED with these messages:`r`n===================================================`r`n" -ForegroundColor Red
}
$jobResult=receive-job -id $job.id
# $jobResults+=,$jobResult
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$jobResult=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computer
stoppedServices='Unknown'
fixedAutomatically=$False
}
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
# Safeguard against stuck jobs
if($timer.elapsed.totalminutes -ge $maxMinutesPerJob){
$expiredJobs=get-job
if($expiredJobs.count -gt 0){
$expiredJobNames=$expiredJobs.Name
$expiredJobs|Remove-Job -Force
write-warning "Shell was aborted to mitigate potential persistency issues due to these expired jobs:`r`n$expiredJobNames"
}else{
write-host "The preset $maxMinutesPerJob minutes have expired. Exiting function."
$null=get-job|Remove-Job -Force
}
break
}
}
$minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
$timer.stop()
write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
return $jobResults #|select -property * -excludeproperty RunspaceId
}
function forceRestartService($serviceName='Spooler'){
$processId=Get-WmiObject -Class Win32_Service -Filter "Name LIKE '$serviceName'"|Select-Object -ExpandProperty ProcessId
if($processId.count -eq 1 -and $processId -ne 0){
try{
$taskkill=taskkill /f /pid $processid
write-host $taskkill
start-service $serviceName
$result=get-service $serviceName
if($result.Status -eq 'Running'){
$processId=Get-WmiObject -Class Win32_Service -Filter "Name LIKE '$serviceName'"|Select-Object -ExpandProperty ProcessId
write-host "$serviceName has successfully restarted with pid $processId" -foregroundcolor GREEN
return $true
}else{
write-host "$serviceName has NOT successfully restarted" -foregroundcolor RED
return $false
}
}catch{
write-warning $_
return $false
}
}else{
try{
start-service $serviceName
}catch{
write-warning $_
# write-warning "$serviceName is not found on $env:computername"
return $null
}
}
}
# Convert text to array
$computernames=@($computerList -split "`n")|%{$_.Trim()}
# Convert credentials from keychain automatically
$credentials=[hashtable]@{}
$keychain.GetEnumerator()|%{$credentials+=[hashtable]@{
$_.Name=New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $_.value.username,$(ConvertTo-SecureString $_.value.Password -AsPlainText -Force)
}}
# Map computernames to credentials
$computerCreds=[hashtable]@{}
foreach($computername in $computernames){
$domain=[regex]::match($computername,'(\.)(.*)').groups[2].value
$credential=$credentials[$domain]
$computerCreds+=[hashtable]@{$computername=$credential}
}
$results=[hashtable]@{}
foreach($computer in $computerCreds.GetEnumerator()){
$pssession=try{new-pssession -computername $computer.name -credential $computer.value}catch{$null}
if($pssession.state -eq 'Opened'){
write-host "Connected to $($computer.name) using $($computer.value.username)"
$result=invoke-command -session $pssession {
param($serviceNames,$forceRestartService)
$services=get-service $serviceNames -EA SilentlyContinue
if($services.Status|?{'Running' -ne $_}){
write-warning "$($services.Name) is/are NOT running on $env:computername"
try{
foreach($service in $services){
if($service.Status -eq 'Starting'){
[scriptblock]::create($forceRestartService).invoke($service.name)
}else{
start-service $service.Name
}
}
# start-service $services.Name
return $true
}catch{
write-warning $_
return $false
}
}else{
write-host "$($services.Name) is/are running on $env:computername" -foregroundcolor Green
return $true
}
} -Args (,$serviceNames)#,${function:forceRestartService}
remove-pssession $pssession
$results+=[hashtable]@{$computer.Name=if($result){'Passed'}else{'Failed'}}
}else{
write-warning "Unable to connect to $($computer.Name)"
$results+=[hashtable]@{$computer.Name='Unknown'}
}
}
$passed=$results.GetEnumerator()|?{$_.Value -eq 'Passed'}
$failed=$results.GetEnumerator()|?{$_.Value -eq 'Failed'}
$unknown=$results.GetEnumerator()|?{$_.Value -eq 'Unknown'}
if($null -ne $passed){
write-host "Passed:`r`n"
$passed|out-string|write-host
}
if($null -ne $failed){
write-host "Failed:`r`n"
$failed|out-string|write-host
}
if($null -ne $unknown){
write-host "Unknowns:`r`n"
$unknown|out-string|write-host
}
February 15, 2023February 15, 2023
0 Comments