PowerShell: Process Watcher – Perform CRM Async Services Maintenance

# invokeCrmAsyncMaintenance.ps1
# Version 0.0.1
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
# Moreover, there's a special handling of the service name 'MSCRMAsyncService$maintenance'

# Obtain credentials stored in environment variables
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;

# List of servers to check
$computerNames=@('CRM1-ASYNC','CRM2-ASYNC')

# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='Async Server Issues'
$smtpRelayServer='relay.kimconnect.com'

function invokeCrmAsyncMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$crmAsyncServiceName='MSCRMAsyncService$maintenance',
        [string]$desiredStatus='running'
    )
    $results=@()
    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            (get-service $stoppedServices.Name).waitforstatus('Running')
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            return [pscustomobject]@{
                stoppedServices=($stoppedServices.Name|out-string).trim()
                failedToStartServices=if($failedServices){($failedServices.Name|out-string).trim()}else{'none'}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                failedToStartServices='none'
            }
        }        
    }    
    function checkService($serviceName,$status='Running'){
        # Sanitation
        $systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        $regexInvalidChars = "[$systemInvalidChars]"
        $serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName|select -first 1
            if($service.Status -eq $status){
                write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                write-host "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            Write-Error $_
            write-host "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }

    $fixAsyncService={
        $erroractionpreference='stop'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){"$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
            write-host 'Restarting Microsoft Dynamics CRM Asynchronous Services...'
            get-service|?{$_.name -like 'MSCRMAsyncService*'}|restart-service
            write-host 'Renewing keys to CRM App server...'
            & "$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe" /R
            if($LASTEXITCODE -ne 0){
                write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
                $result=$false
            }else{
                sleep 3
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                get-service|?{$_.DisplayName -like 'Microsoft Dynamics*(maintenance)'}|restart-service
                $result=$true
                }
            return $result
        }catch{
            write-host $_
            return $false
        }
    }

    foreach ($computerName in $computerNames){
        $session=if($credentials){
                new-pssession -ComputerName $computerName -Credential $credentials
            }else{
                new-pssession -ComputerName $computerName
            }
        if($session.state -eq 'Opened'){
            $serviceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$crmAsyncServiceName)
                [scriptblock]::create($checkService).invoke($crmAsyncServiceName)
                } -args ${function:checkService},$crmAsyncServiceName
            $asyncServiceResult=if ($serviceStatusCode -eq 0){
                    write-host "$crmAsyncServiceName is healthy on $computerName. No actions required." -foregroundcolor Green
                    $true
                }elseif($serviceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }else{
                    write-host "There appears to be a problem with the service status code of $serviceStatusCode"
                    $false
                }
            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            Remove-PSSession $session        
            $results+=[pscustomobject]@{
                computername=$computername
                asyncServiceIsRunning=$asyncServiceResult
                stoppedServices=$otherServicesResult.stoppedServices
                failedToStartServices=$otherServicesResult.failedToStartServices
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $results+=[pscustomobject]@{
                computername=$computername
                asyncServiceIsRunning='Unknown'
                stoppedServices='Unknown'
                failedToStartServices='Unknown'
                }
        }
    }
    return $results    
}

$results=invokeCrmAsyncMaintenance $computerNames $credentials
$problemsDetected=$results|?{$_.stoppedServices -notmatch 'none'}
if($problemsDetected){
    Send-MailMessage -From $emailFrom `
    -To $emailTo `
    -Subject $subject `
    -Body $($problemsDetected|convertto-html -Fragment|out-string) `
    -BodyAsHtml `
    -SmtpServer $smtpRelayServer
}

# crmAsyncMaintenance.ps1
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
# Moreover, there's a special handling of the service name 'MSCRMAsyncService$maintenance'

$computerNames='CRM01','CRM02'

function performCrmAsyncMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$crmAsyncServiceName='MSCRMAsyncService$maintenance',
        [string]$desiredStatus='running'
    )
    $results=@()

    function startAllAutoServices{
        param(
            [string[]]$computerNames=$env:computername,
            [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )
        
        $results=@()
        foreach ($computerName in $computerNames){
            # Start all non-running Auto-Start services
            # $exceptions=($skipServices|%{"'$_'"}) -join ','
            $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
            $nonRunningServices=Get-WmiObject win32_service -ComputerName $computerName -Filter "startmode='auto' AND state!='running' $filterString"
            if($nonRunningServices){
                $nonRunningServices|Invoke-WmiMethod -Name StartService
                # Detect non-running services
                $failedServices = Get-WmiObject win32_service -ComputerName $computerName -Filter "startmode = 'auto' AND state != 'running' $filterString" | select -expand Name
                if($failedServices){
                    write-warning "$computerName`: services with 'stopped' status $failedServices"
                }else{
                    write-host "$computerName`: all auto-start services are running." -ForegroundColor Green
                }
                $results+=[pscustomobject]@{
                    computername=$computerName
                    nonRunningServices=($nonRunningServices.Name|out-string).trim()
                    failedToStart=($failedServices|out-string).trim()
                }
            }else{
                write-host "$computerName all auto-start services are running." -ForegroundColor Green
                $results+=[pscustomobject]@{
                    computername=$computerName
                    nonRunningServices='none'
                    failedToStart='none'
                }
            }
        }
        return $results
    }    
    function checkService($computername=$env:computername,$serviceName,$status='Running'){
    # Sanitation
    #$invalidChars=$processName.IndexOfAny([System.IO.Path]::GetInvalidFileNameChars())
    $systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
    $regexInvalidChars = "[$systemInvalidChars]"
    $serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
    if($desiredStatus -ne 'Stopped'){
        $desiredStatus='Running'
    }
    try{
        $service=get-service -name $serviceName -ComputerName $computername|select -first 1
        if($service.Status -eq $status){
            write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
            return 0
        }elseif($null -ne $service.Status){
            write-host "status doesn`'t match the desired state" -foregroundcolor Red
            return 1
        }else{
            write-host "$serviceName was not found" -foregroundcolor Red
            return -1          
        }
    }catch{
        Write-Error $_
        write-host "$serviceName was not found" -foregroundcolor Red
        return -1
    }
    }

    $action={
        $erroractionpreference='stop'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){"$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
            write-host 'Restarting Microsoft Dynamics CRM Asynchronous Services...'
            get-service|?{$_.name -like 'MSCRMAsyncService*'}|restart-service
            write-host 'Renewing keys to CRM App server...'
            & "$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe" /R
            if($LASTEXITCODE -ne 0){
                write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
                $result=$false
            }else{
                sleep 3
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                get-service|?{$_.DisplayName -like 'Microsoft Dynamics*(maintenance)'}|restart-service
                $result=$true
                }
            return $result
        }catch{
            write-host $_
            return $false
        }
    }
    function performAction($computerName,$serviceStatusCode,$action){
        if ($serviceStatusCode -eq 0){
            write-host "$serviceName is healthy. No actions required." -foregroundcolor Green
            return $true
        }elseif($serviceStatusCode -eq 1){
            $success=invoke-command -ComputerName $computerName -scriptblock $action
            return $success
        }else{
            write-host "There appears to be a problem with the service status code of $serviceStatusCode"
            return $false
        }
    }

    foreach ($computerName in $computerNames){
        $serviceStatusCode=checkService $computerName $crmAsyncServiceName $desiredStatus
        $asyncServiceResult=if($serviceStatusCode -gt 0){performAction $computerName $serviceStatusCode $action}elseif($serviceStatusCode -eq 1){'Error: service not found'}else{$true}
        $otherServicesResult=(startAllAutoServices $computerName $skipServices).failedToStart
        $results+=[pscustomobject]@{
            computername=$computername
            asyncServiceIsRunning=$asyncServiceResult
            failedAutorunServices=$otherServicesResult
            }
    }
    return $results
}

performCrmAsyncMaintenance $computerNames