Version 4:
# invokeCrmServersMaintenance.ps1
# Version 0.0.4
# This script is a processes watcher on Microsoft Dynamics CRM Servers with these routines:
# - Multi-threading to process multiple nodes concurrently to improve efficiency
# - Ensures that all automatically starting services are running
# - Handles of certain special services: 'MSCRMAsyncService$maintenance', 'OneSyncSvc_*'
# - Detects name resolution issues
# - Check for any custom scheduled tasks for signs of failures
# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
# List of servers to check
$computerNames=@(
'CRM-WEB01',
'CRM-SQL01',
'CRM-DEV01'
)
# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
# Other Variables
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$csvFile='C:\scripts\logs\serverIssues.csv'
$limitEmailRecords=100
function invokeCrmServersMaintenance{
param(
[string[]]$computerNames=$env:computername,
[pscredential]$credentials,
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
[string]$desiredStatus='running'
)
$results=@()
$emailRouterServiceName='MSCRMEmail'
$asyncServiceName='MSCRMAsyncService$maintenance'
$targetAsyncServices='MSCRMAsyncService$maintenance','MSCRMAsyncService'
function checkService($serviceName,$status='Running'){
# Sanitation
#$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
#$regexInvalidChars = "[$systemInvalidChars]"
#$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
try{
$service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
if($service.Status -eq $status){
# write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
return 0
}elseif($null -ne $service.Status){
write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
return 1
}else{
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}catch{
#Write-verbose $_
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}
function getFailedScheduledTasks{
param(
$excludedPaths='^\\Microsoft|Mozilla|Integrations\\',
$excludedTaskResults=@(
0, # success
267009, # running
267010, # disabled
267011, # not yet ran
267012, # There are no more runs scheduled for this task
267014, # The last run of the task was terminated by the user
267015, # Either the task has no triggers or the existing triggers are disabled or not set
2147750687, # An instance of this task is already running
3221225786, # The application terminated as a result of a CTRL+C
1073807364, # 40010004 (hex). This means the system cannot open a file 4001 is the facility code. This can safely be ignored as it pertains to CreateExplorerShellUnelevatedTask
2147943517 # Firefox Default Browser Agent
),
$excludedStates=@('Disabled','Running')
)
$windowsVersion=[Environment]::OSVersion.Version
if($windowsVersion -ge [version]'6.2'){
# This function requires Windows 8 / Server 2012 (build 9200) or higher
# Get-ScheduledTask : The term 'Get-ScheduledTask' is not recognized as the name of a cmdlet, function, script file, or
# operable program. Check the spelling of the name, or if a path was included, verify that the path is correct and try
# again.
# At line:1 char:14
# + $customTasks=Get-ScheduledTask|?{ $_.State -ne "Disabled" -and $_.Tas ...
# + ~~~~~~~~~~~~~~~~~
# + CategoryInfo : ObjectNotFound: (Get-ScheduledTask:String) [], CommandNotFoundException
# + FullyQualifiedErrorId : CommandNotFoundException
# $knownStates=@('Unknown','Disabled','Queued','Ready','Running')
# Task status codes: https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-error-and-success-constants
# Common Task Result Codes (int32):
# 0 - The operation completed successfully.
# 1 - Incorrect function called or unknown function called.
# 10 - The environment is incorrect.
# 267008 - Task is ready to run at its next scheduled time.
# 267009 - Task is currently running.
# 267010 - The task will not run at the scheduled times because it has been disabled.
# 267011 - Task has not yet run.
# 267012 - There are no more runs scheduled for this task.
# 267013 - One or more of the properties that are needed to run this task on a schedule have not been set.
# 267014 - The last run of the task was terminated by the user.
# 267015 - Either the task has no triggers or the existing triggers are disabled or not set.
# 2147750671 - Credentials became corrupted.
# 2147750687 - An instance of this task is already running.
# 2147943645 - The service is not available (is "Run only when an user is logged on" checked?).
# 3221225786 - The application terminated as a result of a CTRL+C.
# 3228369022 - Unknown software exception.
$customTasks=Get-ScheduledTask|?{ $_.State -notin $excludedStates -and $_.TaskPath -notmatch $excludedPaths}
$failedTasks=$customTasks|Get-ScheduledTaskInfo|?{$_.LastTaskResult -notin $excludedTaskResults}
if($failedTasks){
return $failedTasks|Select-Object -Property * -ExcludeProperty PSComputerName,CimClass,CimInstanceProperties,CimSystemProperties
}
}
}
function startAllAutoServices{
param(
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
)
$filterString=($skipServices|%{"AND name!='$_' "}) -join ''
$stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
if($stoppedServices){
write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
$null=$stoppedServices|Invoke-WmiMethod -Name StartService
$null=(get-service $stoppedServices.Name).waitforstatus('Running')
# $timeoutSeconds=45
# $timeSpan = New-Object Timespan 0,0,$timeoutSeconds
# foreach($service in $stoppedServices){
# $service = Get-Service $service.Name
# $service.Start()
# $service.WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, $timeSpan)
# }
# Detect failed services
$failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
if($failedServices){
write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
}
$excludedOneSync=$stoppedServices|?{$_.Name -notlike 'OneSyncSvc*'}
$filteredServices=if($excludedOneSync){($excludedOneSync.Name -join ','|out-string).trim()}else{'None'}
return [pscustomobject]@{
stoppedServices=$filteredServices
fixedAutomatically=if($failedServices){$false}else{$true}
}
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
return [pscustomobject]@{
stoppedServices='none'
fixedAutomatically=$true
}
}
}
$fixAsyncService={
$erroractionpreference='continue'
$targetAsyncServices='MSCRMAsyncService$maintenance','MSCRMAsyncService'
try{
$crmTools=if(test-path "$env:programfiles\dynamics 365"){
"$env:programfiles\dynamics 365\Tools"
}else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
$asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
$stoppedAsync=$asyncServices|?{$_.Status -eq 'Stopped'}
if($stoppedAsync){
write-host "Restarting services: $($asyncServices.Name)"
$stoppedAsync|Start-Service
write-host 'Renewing keys to CRM App server...'
Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
}
#cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
# cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
# comment-out because $lastexitcode does not register properly within WinRM sessions
# if($LASTEXITCODE -ne 0){
# write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
# return $false
# }
}catch{
write-host $_
}
return !(get-service $stoppedAsync.Name|?{$_.status -eq 'Stopped'})
}
$fixEmailRouterService={
$erroractionpreference='stop'
$emailRouterServiceName='MSCRMEmail'
$expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
$alreadyRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if(!$alreadyRunning){
try{
$emailRouterXmlFile=if(test-path $expectedXmlLocation){
$expectedXmlLocation
}else{
"${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
}
write-host "Attempting to start email router service normally..."
start-service $emailRouterServiceName
$isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if(!$isRunning){
write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'
$null=stop-service $emailRouterServiceName -force
$null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
start-service $emailRouterServiceName
}
$isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if($isRunning){
return $true
}else{
return $false
}
}catch{
write-host $_
return $false
}
}else{
write-host "$emailRouterServiceName is already running" -ForegroundColor Green
}
}
foreach ($computerName in $computerNames){
$sessionTimeout=New-PSSessionOption -OpenTimeout 120000 # 2 minutes
$sessionIncludePort=New-PSSessionOption -IncludePortInSPN -OpenTimeout 120000
$session=if($credentials){
try{
New-PSSession -ComputerName $computername -Credential $credentials -ea Stop -SessionOption $sessionTimeout
}catch{
New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $sessionIncludePort
}
}else{
try{
New-PSSession -ComputerName $computername -ea Stop -SessionOption $sessionTimeout
}catch{
New-PSSession -ComputerName $computername -SessionOption $sessionIncludePort
}
}
if($session.state -eq 'Opened'){
$asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,[array]$asyncServiceName)
foreach ($x in $asyncServiceName){
$null=$statusCode
$statusCode=[scriptblock]::create($checkService).invoke($x)
if($statusCode){
return $statusCode
}
}
return $statusCode
} -args ${function:checkService},$targetAsyncServices
if($asyncServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$emailRouterServiceName)
[scriptblock]::create($checkService).invoke($emailRouterServiceName)
} -args ${function:checkService},'MSCRMEmail'
if($emailRouterServiceStatusCode -eq 1){
$null=invoke-command -session $session -scriptblock $fixEmailRouterService
}
$otherServicesResult=invoke-command -Session $session -ScriptBlock{
param ($startAllAutoServices,$skipServices)
[scriptblock]::create($startAllAutoServices).invoke($skipServices)
} -args ${function:startAllAutoServices},$skipServices
if($asyncServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$asyncServiceName
}else{
$otherServicesResult.stoppedServices=$asyncServiceName
}
}
if($emailRouterServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+'MSCRMEmail'
}else{
$otherServicesResult.stoppedServices='MSCRMEmail'
}
}
$dnsOK=invoke-command -Session $session -ScriptBlock{
# test-connection google.com -Count 1 -Quiet # This sometimes will freeze the session
$pingResult=ping google.com -n 1
return [bool]($pingResult -match '0% loss')
}
$failedScheduledTasks=invoke-command -Session $session -ScriptBlock{
param($x)
[scriptblock]::create($x).invoke()
} -Args ${function:getFailedScheduledTasks}
Remove-PSSession $session
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$otherStoppedServices=$otherServicesResult.stoppedServices
$otherStoppedServices=.{
$x=$otherStoppedServices|?{$_ -ne 'none'}
if(!$dnsOK){
$x+=,'dnsClient'
}
if($failedScheduledTasks){
$x+=,"FailedScheduleTasks: $($failedScheduledTasks.TaskName -join ',')"
}
return $x
}
$stoppedServices=($otherStoppedServices|out-string).trim()
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices=if($stoppedServices){$stoppedServices}else{'None'}
fixedAutomatically=if(!$failedScheduledTasks -and $dnsOK -and $otherServicesResult.fixedAutomatically){$True}else{$False}
}
}else{
write-warning "$env:computername cannnot connect to $computername via WinRM"
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices='Unknown'
fixedAutomatically='Unknown'
}
}
}
return $results
}
function checkCrmServices($computerNames,$credentials,$skipServices,$verbose=$true){
$timer=[System.Diagnostics.Stopwatch]::StartNew()
$jobResults=@()
$lineBreak=60
$dotCount=0
$minute=0
$processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
$cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
$maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
foreach($computerName in $computerNames){
$thisIterationCompleted=$false
do {
$jobsCount=(Get-Job -State 'Running').Count
if ($jobsCount -lt $maxSimultaneousJobs){
if($verbose){write-host "Initiating job for $computerName"}
$null=Start-Job -name $computerName -ScriptBlock {
param($invokeCrmServersMaintenance,$computerName,$credentials,$skipServices)
[scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$skipServices)
} -Args ${function:invokeCrmServersMaintenance},$computerName,$credentials,$skipServices
$thisIterationCompleted=$true
}else{
if($verbose){
if($dotCount++ -lt $lineBreak){
write-host '.' -NoNewline
}else{
$minute++
write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
$dotCount=0
}
}
sleep -seconds 1
}
}until ($thisIterationCompleted)
}
$totalJobsCount=(get-job).count
$processedCount=0
while($processedCount -lt $totalJobsCount){
$completedJobs=get-job|?{$_.State -eq 'Completed'}
if($completedJobs){
foreach ($job in $completedJobs){
$computer=$job.Name
if($verbose){
write-host "`r`n===================================================`r`n$computer job completed with these messages:`r`n===================================================`r`n"
}
$jobResult=receive-job -id $job.id
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
}
$minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
$timer.stop()
write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
return $jobResults #|select -property * -excludeproperty RunspaceId
}
$results=checkCrmServices $computerNames $credentials $skipServices $false|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically
write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
$previousProblems=import-csv -path $csvFile|select -last $problemsDetected.computerName.count|select -Property * -ExcludeProperty timeStamp
if(!(test-path $csvFile)){
if(!(test-path $(split-path $csvFile -parent))){mkdir $(split-path $csvFile -parent) -force}
$header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
Add-Content -Path $csvFile -Value $header
}
$problemsDetected|Export-Csv -Path $csvFile -Append
$problemsDetectedReformatted=$problemsDetected|ConvertTo-Csv|convertFrom-csv|select -Property * -ExcludeProperty timeStamp
$sameProblemAsPrior=!(Compare-Object $problemsDetectedReformatted.PSObject.Properties $previousProblems.PSObject.Properties)
if(!$sameProblemAsPrior){
$css="
<style>
.h1 {
font-size: 18px;
height: 40px;
padding-top: 80px;
margin: auto;
text-align: center;
}
.h5 {
font-size: 22px;
text-align: center;
}
.th {text-align: center;}
.table {
padding:7px;
border:#4e95f4 1px solid;
background-color: white;
margin-left: auto;
margin-right: auto;
width: 100%
}
.colgroup {}
.th { background: #0046c3; color: #fff; padding: 5px 10px; }
.td { font-size: 11px; padding: 5px 20px; color: #000;
width: 1px;
white-space: pre;
}
.tr { background: #b8d1f3;}
.tr:nth-child(even) {
background: #dae5f4;
width: 1%;
white-space: nowrap
}
.tr:nth-child(odd) {
background: #b8d1f3;
width: 1%;
white-space: nowrap
}
</style>
"
$currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
$historicalReport=Import-CSV $csvFile|sort timeStamp -Descending|select-object -first $limitEmailRecords|ConvertTo-Html -fragment|Out-String
$currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
write-host "Updating the report html file: $csvFile.html"
$null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$csvFile.html"
Send-MailMessage -From $emailFrom `
-To $emailTo `
-Subject $subject `
-Body $emailContent `
-BodyAsHtml `
-SmtpServer $smtpRelayServer
}else{
write-host "Current issue is same as before. Email sending is skipped"
}
}
Version 3:
# invokeCrmServersMaintenance.ps1
# Version 0.0.3
# This script is a processes watcher on Microsoft Dynamics CRM Servers with these routines:
# - Ensures that all automatically starting services are running
# - Handles of certain special services: 'MSCRMAsyncService$maintenance', 'OneSyncSvc_*'
# - Detects name resolution issues
# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
# List of servers to check
$computerNames=@(
'CRM-WEB01',
'CRM-SQL01',
'CRM-DEV01'
)
# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$logFile='C:\scripts\logs\ServersMaintenanceIssues.csv'
$limitEmailRecords=100
function invokeCrmServersMaintenance{
param(
[string[]]$computerNames=$env:computername,
[pscredential]$credentials,
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
[string]$desiredStatus='running'
)
$results=@()
$emailRouterServiceName='MSCRMEmail'
$asyncServiceName='MSCRMAsyncService$maintenance'
function checkService($serviceName,$status='Running'){
# Sanitation
#$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
#$regexInvalidChars = "[$systemInvalidChars]"
#$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
try{
$service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
if($service.Status -eq $status){
write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
return 0
}elseif($null -ne $service.Status){
write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
return 1
}else{
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}catch{
#Write-verbose $_
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}
function startAllAutoServices{
param(
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
)
$filterString=($skipServices|%{"AND name!='$_' "}) -join ''
$stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
if($stoppedServices){
write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
$null=$stoppedServices|Invoke-WmiMethod -Name StartService
(get-service $stoppedServices.Name).waitforstatus('Running')
# Detect failed services
$failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
if($failedServices){
write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
}
$excludeOneSync=$stoppedServices|?{$_.Name -notlike 'OneSyncSvc*'}
$filteredServices=if($excludeOneSync){$excludeOneSync}else{'None'}
return [pscustomobject]@{
stoppedServices=($filteredServices.Name -join ','|out-string).trim()
fixedAutomatically=if($failedServices){$false}else{$true}
}
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
return [pscustomobject]@{
stoppedServices='none'
fixedAutomatically=$true
}
}
}
$fixAsyncService={
$erroractionpreference='stop'
$asyncServiceName='MSCRMAsyncService$maintenance'
try{
$crmTools=if(test-path "$env:programfiles\dynamics 365"){
"$env:programfiles\dynamics 365\Tools"
}else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
$asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
write-host "Restarting services: $($asyncServices.Name)"
restart-service $asyncServices
write-host 'Renewing keys to CRM App server...'
Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
#cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
# cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
# comment-out because $lastexitcode does not register properly within WinRM sessions
# if($LASTEXITCODE -ne 0){
# write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
# return $false
# }else{
sleep 1
write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
restart-service $asyncServiceName
return $true
# }
}catch{
write-host $_
return $false
}
}
$fixEmailRouterService={
$erroractionpreference='stop'
$emailRouterServiceName='MSCRMEmail'
$expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
try{
$emailRouterXmlFile=if(test-path $expectedXmlLocation){
$expectedXmlLocation
}else{
"${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
}
write-host "Attempting to start email router service normally..."
start-service $emailRouterServiceName
$isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if(!$isRunning){
write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'
stop-service $emailRouterServiceName -force
$null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
start-service $emailRouterServiceName
}
return $true
}catch{
write-host $_
return $false
}
}
foreach ($computerName in $computerNames){
$session=if($credentials){
try{
New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
}catch{
New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}else{
try{
New-PSSession -ComputerName $computername -ea Stop
}catch{
New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}
if($session.state -eq 'Opened'){
$asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$asyncServiceName)
[scriptblock]::create($checkService).invoke($asyncServiceName)
} -args ${function:checkService},$asyncServiceName
if($asyncServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$emailRouterServiceName)
[scriptblock]::create($checkService).invoke($emailRouterServiceName)
} -args ${function:checkService},$emailRouterServiceName
if($emailRouterServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixEmailRouterService
}
$otherServicesResult=invoke-command -Session $session -ScriptBlock{
param ($startAllAutoServices,$skipServices)
[scriptblock]::create($startAllAutoServices).invoke($skipServices)
} -args ${function:startAllAutoServices},$skipServices
if($asyncServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$asyncServiceName
}else{
$otherServicesResult.stoppedServices=$asyncServiceName
}
}
if($emailRouterServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$emailRouterServiceName
}else{
$otherServicesResult.stoppedServices=$emailRouterServiceName
}
}
$dnsOK=invoke-command -Session $session -ScriptBlock{test-connection google.com -Count 1 -Quiet}
Remove-PSSession $session
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$otherStoppedServices=$otherServicesResult.stoppedServices
$otherStoppedServices=.{if(!$dnsOK){
$x=$otherStoppedServices|?{$_ -ne 'none'}
$x+=,'dnsClient'
return $x
}else{
return $otherStoppedServices}}
$stoppedServices=($otherStoppedServices|out-string).replace("`n",'').trim()
$fixedAutomatically=if($dnsOK){$otherServicesResult.fixedAutomatically}else{$false}
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices=$stoppedServices
fixedAutomatically=$fixedAutomatically
}
}else{
write-warning "$env:computername cannnot connect to $computername via WinRM"
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices='Unknown'
fixedAutomatically='Unknown'
}
}
}
return $results
}
$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically # This last part is necessary to remove invoke-command junks
write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
if(!(test-path $logFile)){
if(!(test-path $(split-path $logFile -parent))){mkdir $(split-path $logFile -parent) -force}
$header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
Add-Content -Path $logFile -Value $header
}
$problemsDetected|Export-Csv -Path $logfile -Append
$css=@"
<style>
.h1 {
font-size: 18px;
height: 40px;
padding-top: 80px;
margin: auto;
text-align: center;
}
.h5 {
font-size: 22px;
text-align: center;
}
.th {text-align: center;}
.table {
padding:7px;
border:#4e95f4 1px solid;
background-color: white;
margin-left: auto;
margin-right: auto;
width: 100%
}
.colgroup {}
.th { background: #0046c3; color: #fff; padding: 5px 10px; }
.td { font-size: 11px; padding: 5px 20px; color: #000;
width: 1px;
white-space: pre;
}
.tr { background: #b8d1f3;}
.tr:nth-child(even) {
background: #dae5f4;
width: 1%;
white-space: nowrap
}
.tr:nth-child(odd) {
background: #b8d1f3;
width: 1%;
white-space: nowrap
}
</style>
"@
$currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
$historicalReport=Import-CSV $logFile|sort timeStamp -Descending|select-object -first $limitEmailRecords|ConvertTo-Html -fragment|Out-String
$currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
write-host "Updating the report html file: $logFile.html"
$null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$logFile.html"
Send-MailMessage -From $emailFrom `
-To $emailTo `
-Subject $subject `
-Body $emailContent `
-BodyAsHtml `
-SmtpServer $smtpRelayServer
}
Version 2:
# invokeCrmServersMaintenance.ps1
# Version 0.0.2
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
# Moreover, there's a special handling of the service name 'MSCRMAsyncService$maintenance'
# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
# List of servers to check
$computerNames=@(
'CRM-WEB01',
'CRM-SQL01',
'CRM-DEV01'
)
# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$logFile='C:\scripts\logs\ServersMaintenanceIssues.csv'
function invokeCrmServersMaintenance{
param(
[string[]]$computerNames=$env:computername,
[pscredential]$credentials,
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
[string]$desiredStatus='running'
)
$results=@()
$emailRouterServiceName='MSCRMEmail'
$asyncServiceName='MSCRMAsyncService$maintenance'
function checkService($serviceName,$status='Running'){
# Sanitation
#$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
#$regexInvalidChars = "[$systemInvalidChars]"
#$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
try{
$service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
if($service.Status -eq $status){
write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
return 0
}elseif($null -ne $service.Status){
write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
return 1
}else{
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}catch{
#Write-verbose $_
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}
function startAllAutoServices{
param(
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
)
$filterString=($skipServices|%{"AND name!='$_' "}) -join ''
$stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
if($stoppedServices){
write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
$null=$stoppedServices|Invoke-WmiMethod -Name StartService
(get-service $stoppedServices.Name).waitforstatus('Running')
# Detect failed services
$failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
if($failedServices){
write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
}
return [pscustomobject]@{
stoppedServices=($stoppedServices.Name -join ','|out-string).trim()
fixedAutomatically=if($failedServices){$false}else{$true}
}
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
return [pscustomobject]@{
stoppedServices='none'
fixedAutomatically=$true
}
}
}
$fixAsyncService={
$erroractionpreference='stop'
$asyncServiceName='MSCRMAsyncService$maintenance'
try{
$crmTools=if(test-path "$env:programfiles\dynamics 365"){
"$env:programfiles\dynamics 365\Tools"
}else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
$asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
write-host "Restarting services: $($asyncServices.Name)"
restart-service $asyncServices
write-host 'Renewing keys to CRM App server...'
Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
#cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
# cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
# comment-out because $lastexitcode does not register properly within WinRM sessions
# if($LASTEXITCODE -ne 0){
# write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
# return $false
# }else{
sleep 1
write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
restart-service $asyncServiceName
return $true
# }
}catch{
write-host $_
return $false
}
}
$fixEmailRouterService={
$erroractionpreference='stop'
$emailRouterServiceName='MSCRMEmail'
$expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
try{
$emailRouterXmlFile=if(test-path $expectedXmlLocation){
$expectedXmlLocation
}else{
"${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
}
write-host "Attempting to start email router service normally..."
start-service $emailRouterServiceName
$isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if(!$isRunning){
write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'
stop-service $emailRouterServiceName -force
$null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
start-service $emailRouterServiceName
}
return $true
}catch{
write-host $_
return $false
}
}
foreach ($computerName in $computerNames){
$session=if($credentials){
try{
New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
}catch{
New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}else{
try{
New-PSSession -ComputerName $computername -ea Stop
}catch{
New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}
if($session.state -eq 'Opened'){
$asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$asyncServiceName)
[scriptblock]::create($checkService).invoke($asyncServiceName)
} -args ${function:checkService},$asyncServiceName
if($asyncServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$emailRouterServiceName)
[scriptblock]::create($checkService).invoke($emailRouterServiceName)
} -args ${function:checkService},$emailRouterServiceName
if($emailRouterServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixEmailRouterService
}
$otherServicesResult=invoke-command -Session $session -ScriptBlock{
param ($startAllAutoServices,$skipServices)
[scriptblock]::create($startAllAutoServices).invoke($skipServices)
} -args ${function:startAllAutoServices},$skipServices
if($asyncServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$asyncServiceName
}else{
$otherServicesResult.stoppedServices=$asyncServiceName
}
}
if($emailRouterServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$emailRouterServiceName
}else{
$otherServicesResult.stoppedServices=$emailRouterServiceName
}
}
Remove-PSSession $session
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices=($otherServicesResult.stoppedServices|out-string).replace("`n",'').trim()
fixedAutomatically=$otherServicesResult.fixedAutomatically
}
}else{
write-warning "$env:computername cannnot connect to $computername via WinRM"
$timeZoneName=[System.TimeZoneInfo]::Local.StandardName
$abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
$timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
$timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
$results+=[pscustomobject]@{
timeStamp=$timeStamp
computerName=$computerName
stoppedServices='Unknown'
fixedAutomatically='Unknown'
}
}
}
return $results
}
$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically # This last part is necessary to remove invoke-command junks
write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
if(!(test-path $logFile)){
if(!(test-path $(split-path $logFile -parent))){mkdir $(split-path $logFile -parent) -force}
$header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
Add-Content -Path $logFile -Value $header
}
$problemsDetected|Export-Csv -Path $logfile -Append
$css=@"
<style>
.h1 {
font-size: 18px;
height: 40px;
padding-top: 80px;
margin: auto;
text-align: center;
}
.h5 {
font-size: 22px;
text-align: center;
}
.th {text-align: center;}
.table {
padding:7px;
border:#4e95f4 1px solid;
background-color: white;
margin-left: auto;
margin-right: auto;
width: 100%
}
.colgroup {}
.th { background: #0046c3; color: #fff; padding: 5px 10px; }
.td { font-size: 11px; padding: 5px 20px; color: #000;
width: 1px;
white-space: pre;
}
.tr { background: #b8d1f3;}
.tr:nth-child(even) {
background: #dae5f4;
width: 1%;
white-space: nowrap
}
.tr:nth-child(odd) {
background: #b8d1f3;
width: 1%;
white-space: nowrap
}
</style>
"@
$currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
$historicalReport=Import-CSV $logFile | sort timeStamp -Descending | ConvertTo-Html -fragment | Out-String
$currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
$emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
write-host "Updating the report html file: $logFile.html"
$null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$logFile.html"
Send-MailMessage -From $emailFrom `
-To $emailTo `
-Subject $subject `
-Body $emailContent `
-BodyAsHtml `
-SmtpServer $smtpRelayServer
}
Version Multi-threading Test (Slower than Version 2):
# invokeCrmServersMaintenance.ps1
# Version 0.0.3
# This script is a processes watcher on CRM Servers:
# a. Ensures that all automatically starting services are running
# b. Special handling of delicate services: 'MSCRMAsyncService$maintenance' and 'MSCRMEmail'
# c. Simultaneus executions by dynamically limiting concurrent jobs depending on available CPU cores
# Obtain credentials being passed by Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
# List of servers to check
$computerNames=@(
'CRM-WEB01',
'CRM-SQL01',
'CRM-DEV01'
)
# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
function invokeCrmServersMaintenance{
param(
[string[]]$computerNames=$env:computername,
[pscredential]$credentials,
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
[string]$desiredStatus='running'
)
$results=@()
$emailRouterServiceName='MSCRMEmail'
$asyncServiceName='MSCRMAsyncService$maintenance'
function checkService($serviceName,$status='Running'){
# Sanitation
#$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
#$regexInvalidChars = "[$systemInvalidChars]"
#$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
try{
$service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
if($service.Status -eq $status){
write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
return 0
}elseif($null -ne $service.Status){
write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
return 1
}else{
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}catch{
#Write-verbose $_
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}
function startAllAutoServices{
param(
[string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
)
$filterString=($skipServices|%{"AND name!='$_' "}) -join ''
$stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
if($stoppedServices){
write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
$null=$stoppedServices|Invoke-WmiMethod -Name StartService
(get-service $stoppedServices.Name).waitforstatus('Running')
# Detect failed services
$failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
if($failedServices){
write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
}
return [pscustomobject]@{
stoppedServices=($stoppedServices.Name -join ','|out-string).trim()
allServicesNowRunning=if($failedServices){$false}else{$true}
}
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
return [pscustomobject]@{
stoppedServices='none'
allServicesNowRunning=$true
}
}
}
$fixAsyncService={
$erroractionpreference='stop'
$asyncServiceName='MSCRMAsyncService$maintenance'
try{
$crmTools=if(test-path "$env:programfiles\dynamics 365"){
"$env:programfiles\dynamics 365\Tools"
}else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
$asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
write-host "Restarting services: $($asyncServices.Name)"
restart-service $asyncServices
write-host 'Renewing keys to CRM App server...'
Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
#cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
# cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
# comment-out because $lastexitcode does not register properly within WinRM sessions
# if($LASTEXITCODE -ne 0){
# write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
# return $false
# }else{
sleep 1
write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
restart-service $asyncServiceName
return $true
# }
}catch{
write-host $_
return $false
}
}
$fixEmailRouterService={
$erroractionpreference='stop'
$emailRouterServiceName='MSCRMEmail'
$expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
try{
$emailRouterXmlFile=if(test-path $expectedXmlLocation){
$expectedXmlLocation
}else{
"${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
}
write-host "Attempting to start email router service normally..."
start-service $emailRouterServiceName
$isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
if(!$isRunning){
write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'
stop-service $emailRouterServiceName -force
$null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
start-service $emailRouterServiceName
}
return $true
}catch{
write-host $_
return $false
}
}
foreach ($computerName in $computerNames){
$session=if($credentials){
try{
New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
}catch{
New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}else{
try{
New-PSSession -ComputerName $computername -ea Stop
}catch{
New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}
if($session.state -eq 'Opened'){
$asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$asyncServiceName)
[scriptblock]::create($checkService).invoke($asyncServiceName)
} -args ${function:checkService},$asyncServiceName
if($asyncServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$emailRouterServiceName)
[scriptblock]::create($checkService).invoke($emailRouterServiceName)
} -args ${function:checkService},$emailRouterServiceName
if($emailRouterServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixEmailRouterService
}
$otherServicesResult=invoke-command -Session $session -ScriptBlock{
param ($startAllAutoServices,$skipServices)
[scriptblock]::create($startAllAutoServices).invoke($skipServices)
} -args ${function:startAllAutoServices},$skipServices
if($asyncServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$asyncServiceName
}else{
$otherServicesResult.stoppedServices=$asyncServiceName
}
}
if($emailRouterServiceStatusCode -eq 1){
if($otherServicesResult.stoppedServices -ne 'none'){
$otherServicesResult.stoppedServices+=','+$emailRouterServiceName
}else{
$otherServicesResult.stoppedServices=$emailRouterServiceName
}
}
Remove-PSSession $session
$results+=[pscustomobject]@{
computerName=$computerName
stoppedServices=($otherServicesResult.stoppedServices|out-string).replace("`n",'').trim()
allServicesNowRunning=$otherServicesResult.allServicesNowRunning
timeStamp=(get-date|out-string).trim()
}
}else{
write-warning "$env:computername cannnot connect to $computername via WinRM"
$results+=[pscustomobject]@{
computerName=$computerName
stoppedServices='Unknown'
allServicesNowRunning='Unknown'
timeStamp=(get-date|out-string).trim()
}
}
}
return $results
}
#$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property computername,stoppedServices,allServicesNowRunning # This last part is necessary to remove invoke-command junks
function checkCrmServices($computerNames,$credentials,$skipServices){
$timer=[System.Diagnostics.Stopwatch]::StartNew()
$jobResults=@()
$lineBreak=60
$dotCount=0
$minute=0
$processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
$cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
$maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
foreach($computerName in $computerNames){
$thisIterationCompleted=$false
do {
$jobsCount=(Get-Job -State 'Running').Count
if ($jobsCount -lt $maxSimultaneousJobs){
write-host "Initiating job for $computerName"
$null=Start-Job -name $computerName -ScriptBlock {
param($invokeCrmServersMaintenance,$computerName,$credentials,$skipServices)
[scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$skipServices)
} -Args ${function:invokeCrmServersMaintenance},$computerName,$credentials,$skipServices
$thisIterationCompleted=$true
}else{
if($dotCount++ -lt $lineBreak){
write-host '.' -NoNewline
}else{
$minute++
write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
$dotCount=0
}
sleep -seconds 1
}
}until ($thisIterationCompleted)
}
$totalJobsCount=(get-job).count
$processedCount=0
while($processedCount -lt $totalJobsCount){
$completedJobs=get-job|?{$_.State -eq 'Completed'}
if($completedJobs){
foreach ($job in $completedJobs){
$computer=$job.Name
write-host "`r`n===================================================`r`n$computer job completed with these messages:`r`n===================================================`r`n"
$jobResult=receive-job -id $job.id
$jobResults+=,$jobResult
remove-job -id $job.id -force
$processedCount++
}
}
}
$minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
$timer.stop()
write-host "This process has completed in $minutesElapsed minutes."
return $jobResults
}
$results=checkCrmServices $computerNames $credentials $skipServices
write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
Send-MailMessage -From $emailFrom `
-To $emailTo `
-Subject $subject `
-Body $($problemsDetected|convertto-html -Fragment|out-string) `
-BodyAsHtml `
-SmtpServer $smtpRelayServer
}
Version 1 (buggy):
# invokeCrmServersMaintenance.ps1
# Version 0.0.1
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
# Obtain credentials being passed by Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
# List of servers to check
$computerNames=@(
'CRM-WEB01',
'CRM-SQL01',
'CRM-DEV01'
)
# Email relay parameters
$emailFrom='[email protected]'
$emailTo='[email protected]'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
# Services to skip
$skipServices=@('BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
function invokeCrmServersMaintenance{
param(
[string[]]$computerNames=$env:computername,
[pscredential]$credentials,
[string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
[string]$desiredStatus='running'
)
$results=@()
$emailRouterServiceName='MSCRMEmail'
$asyncServiceName='MSCRMAsyncService$maintenance'
function checkService($serviceName,$status='Running'){
# Sanitation
#$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
#$regexInvalidChars = "[$systemInvalidChars]"
#$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
try{
$service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
if($service.Status -eq $status){
write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
return 0
}elseif($null -ne $service.Status){
write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
return 1
}else{
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}catch{
#Write-verbose $_
#write-verbose "$serviceName was not found" -foregroundcolor Red
return -1
}
}
function startAllAutoServices{
param(
[string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
)
$filterString=($skipServices|%{"AND name!='$_' "}) -join ''
$stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
if($stoppedServices){
write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
$null=$stoppedServices|Invoke-WmiMethod -Name StartService
(get-service $stoppedServices.Name).waitforstatus('Running')
# Detect failed services
$failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
if($failedServices){
write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
}
return [pscustomobject]@{
stoppedServices=($stoppedServices.Name|out-string).trim()
failedToStartServices=if($failedServices){($failedServices.Name|out-string).trim()}else{'none'}
}
}else{
write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
return [pscustomobject]@{
stoppedServices='none'
failedToStartServices='none'
}
}
}
$fixAsyncService={
$erroractionpreference='stop'
try{
$crmTools=if(test-path "$env:programfiles\dynamics 365"){"$env:programfiles\dynamics 365\Tools"
}else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
write-host 'Restarting Microsoft Dynamics CRM Asynchronous Services...'
get-service|?{$_.name -like 'MSCRMAsyncService*'}|restart-service
write-host 'Renewing keys to CRM App server...'
& "$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe" /R
if($LASTEXITCODE -ne 0){
write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
$result=$false
}else{
sleep 3
write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
get-service|?{$_.DisplayName -like 'Microsoft Dynamics*(maintenance)'}|restart-service
$result=$true
}
return $result
}catch{
write-host $_
return $false
}
}
$fixEmailRouterService={
$erroractionpreference='stop'
$emailRouterServiceName='MSCRMEmail'
$expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
try{
$emailRouterXmlFile=if(test-path $expectedXmlLocation){
$expectedXmlLocation
}else{
"${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
}
write-host 'Restarting Microsoft Dynamics CRM Email Router Services...'
stop-service $emailRouterServiceName -force
$null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
start-service $emailRouterServiceName
return $true
}catch{
write-host $_
return $false
}
}
foreach ($computerName in $computerNames){
$session=if($credentials){
try{
New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
}catch{
New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}else{
try{
New-PSSession -ComputerName $computername -ea Stop
}catch{
New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
}
}
if($session.state -eq 'Opened'){
$asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$asyncServiceName)
[scriptblock]::create($checkService).invoke($asyncServiceName)
} -args ${function:checkService},$asyncServiceName
if($asyncServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
param ($checkService,$emailRouterServiceName)
[scriptblock]::create($checkService).invoke($emailRouterServiceName)
} -args ${function:checkService},$emailRouterServiceName
if($emailRouterServiceStatusCode -eq 1){
invoke-command -session $session -scriptblock $fixAsyncService
}
$otherServicesResult=invoke-command -Session $session -ScriptBlock{
param ($startAllAutoServices,$skipServices)
[scriptblock]::create($startAllAutoServices).invoke($skipServices)
} -args ${function:startAllAutoServices},$skipServices
if($asyncServiceStatusCode -eq 1){
$otherServicesResult.stoppedServices+=$crmAsyncServiceName
}
if($emailRouterServiceStatusCode -eq 1){
$otherServicesResult.stoppedServices+=$emailRouterServiceName
}
Remove-PSSession $session
$results+=[pscustomobject]@{
computername=$computername
#asyncServiceIsRunning=$asyncServiceResult
stoppedServices=$otherServicesResult.stoppedServices
failedToStartServices=$otherServicesResult.failedToStartServices
}
}else{
write-warning "$env:computername cannnot connect to $computername via WinRM"
$results+=[pscustomobject]@{
computername=$computername
#asyncServiceIsRunning='Unknown'
stoppedServices='Unknown'
failedToStartServices='Unknown'
}
}
}
return $results
}
$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices
$problemsDetected=$results|?{$_.stoppedServices -notmatch 'none'}
if($problemsDetected){
Send-MailMessage -From $emailFrom `
-To $emailTo `
-Subject $subject `
-Body $($problemsDetected|convertto-html -Fragment|out-string) `
-BodyAsHtml `
-SmtpServer $smtpRelayServer
}
Categories: