The script below is for informational purposes. Here is a better version
<#
.Description FileCopyScript_v1.1.8.ps1
Note: this is legacy code. The succeding complete rewrite is called "KimConnectCopy," a GPL script posted on KimConnect.com
Please be advised that Emcopy.exe is a product of EMC Corporation. Please ensure that you have agreed to their licensing prior to use.
Purpose: this PowerShell Script is to efficiently mirror large batches of files using Emcopy in conjunction with Volume Shadow Services
Current Features:
0. Validate the sources and destinations (array/object) and remove invalid items
1. Create a snapshot of the source volume using Shadow Copy to capture any locked files
2. Execute emcopy to mirror the source to its corresponding destination
3. Output a log of the file copy operation
4. Sample the copied files to 'spot check' any time stamp variances
5. Execute in the context of an Administrator
6. Search newly generated log for any entries resembling the keyword "ERROR" and output an errors log
7. On "final sync," disable and disconnect SMB sessions at the source to quiescent files before running a copy operation.
Features planned for development:
1. Enable Volume Shadow Copy (VSS) at Source machines if it has been disabled, and reverse the action when done with copying
2. Trigger Remote Powershell to launch execution from a middle server (a "jump box" that is not a source nor destination)
if the provided Source is detected as a Universal Naming Convention (UNC) path instead of a local file system (LFS) path
Limitations:
1. This iteration requires that script is triggered from a local Windows machine with Internet access (no proxies) to download dependencies.
2. Sources must be LFS to work with VSS and Destinations could either be LFS or UNC paths.
Troubleshooting:
# Create snapshot
$targetVolume="M:\"
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy";
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible");
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID };
$thisShadowPath = $thisShadow.DeviceObject + "\";
C:
cmd /c mklink /d "C:\shadowcopy" $thisShadowPath
# Remove snapshot
$thisShadow.Delete()
$thisSnapshot.Dispose()
(Get-Item "C:\shadowcopy").Delete()
# Delete all shadows
vssadmin delete shadows /all /Quiet
# Delete snapshots on a volume
$targetVolume="X:"
vssadmin delete shadows /For=$targetVolume /Quiet
#>
# Specify Sources (LFS) and Destinations (UNC)
# Using System.Array Object[] constructor to create a two dimensional Array
$arr=@();
$arr+=[PSCustomObject]@{Clustername='CLUSTER1';From='H:\Users';To='\\FILESERVER03\Users'}; # Clustername indicates that this source is part of a Microsoft failover cluster
$arr+=[PSCustomObject]@{Clustername='';From='C:\AdminDownloads';To='\\FILESERVER03\AdminDownloads'}; # Clustername = $null means that this item is originating from a standalone file server
# Init global variables
$GLOBAL:totalTime=0;
$GLOBAL:totalStorage=0;
$GLOBAL:sampleSize=1000;
$clusterSizeOfNetworkShare=16384;
$GLOBAL:shadowMount="C:\shadowcopy";
$checkFileHashes=$false; #PS 4.0 required
$compareDirectories=$true;
$toValidateInputs=$false;
$toSampleTimeStamp=$true;
[decimal]$powerShellVersion="$($PSVersionTable.PSVersion.Major)`.$($PSVersionTable.PSVersion.Minor)";
$calculateSizeOnDisk=$false;
$finalSync=$true;
# Initialize log files
$hostname=$ENV:computername
$dateStamp = Get-Date -Format "yyyy-MM-dd-hhmmss"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath`\filecopy_logs\$hostname"
$logFile="$logPath`\$dateStamp.txt"
$errorsLogFile="$logPath`\$hostname-filecopy-errors-$dateStamp.txt"
$log="/LOG+:$logFile"
$lockedFilesReport="$logPath`\$hostname-locked-files-log-$dateStamp.txt"
$pathErrorsLog="$logPath`\$hostname-path-errors-log-$dateStamp.txt"
# Emcopy switches
$useEmCopy=$true;
$emcopySwitches="/o /secforce /d /c /r:0 /w:0 /th 32 /s /purge /sdd /stream $log";
$quickEmcopySwitches="/de /s /purge /r:0 /w:0 /c $log";
<# Switch explanations
/s copies sub directories
/purge removes files and directories from the destination that do not exist in the source.
/sdd forces the target directories dates to be synchronized with the source directory.
/de Compares both file size and last modification time when deciding to update a file, updates it if either have been changed.
/cm md5 - checks the file content after copying using and md5 comparison of the source and destination.
/o copies the files owner, without this the account used for the copy will be the owner
/secforce overwrites the destination security settings with the source security settings (no merging, actual update of security settings at the destination)
/sd preserves security, the file isn't copied if an error occurs during security settings.
/th 128 - Uses 128 threads, default is 64
/r:0 retries zero times
/w:0 is the wait time in seconds between retries
/c will allow the process to continue after the retries
/log:filename option allows to redirect the console messages to a new file.
/log+:filename option appends the new messages to an existing file.
/stream option enables the copie of files and directories datastreams. Without that option only the main datastream of files are copied.
#>
#$robocopySwitches="/TBD /FFT /NS /NC /NDL /S /E /COPY:DATS /DCOPY:T /PURGE /MIR /NP /XO /XJF /XJD /R:0 /W:0 $log";
$robocopySwitches="/TBD /FFT /NS /NC /NDL /S /E /COPY:DATS /DCOPY:T /PURGE /MIR /B /NP /XO /XJF /XJD /R:0 /W:0 $log";
################################## Excuting Program as an Administrator ####################################
# Get the ID and security principal of the current user account
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)
# Get the security principal for the Administrator role
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator
# Check to see if we are currently running "as Administrator"
if ($myWindowsPrincipal.IsInRole($adminRole))
{
# We are running "as Administrator" - so change the title and background color to indicate this
$Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)"
#$Host.UI.RawUI.BackgroundColor = "Black"
clear-host
}
else
{
# We are not running "as Administrator" - so relaunch as administrator
# Create a new process object that starts PowerShell
$newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell";
# Specify the current script path and name as a parameter
$newProcess.Arguments = $myInvocation.MyCommand.Definition;
# Indicate that the process should be elevated
$newProcess.Verb = "runas";
# Start the new process
[System.Diagnostics.Process]::Start($newProcess);
# Exit from the current, unelevated, process
exit
}
Write-Host -NoNewLine "Running as Administrator..."
################################## Excuting Program as an Administrator ####################################
################################## Add System Backup Privileges ####################################
function addSystemPrivilege{
param(
[String[]]$privileges=@("SeBackupPrivilege","SeRestorePrivilege")
)
function includeSystemPrivileges{
$win32api = @'
using System;
using System.Runtime.InteropServices;
namespace SystemPrivilege.Win32API
{
[StructLayout(LayoutKind.Sequential)]
public struct LUID
{
public UInt32 LowPart;
public Int32 HighPart;
}
[StructLayout(LayoutKind.Sequential)]
public struct LUID_AND_ATTRIBUTES
{
public LUID Luid;
public UInt32 Attributes;
}
[StructLayout(LayoutKind.Sequential)]
public struct TOKEN_PRIVILEGES
{
public UInt32 PrivilegeCount;
public LUID Luid;
public UInt32 Attributes;
}
public class Privileges
{
public const UInt32 DELETE = 0x00010000;
public const UInt32 READ_CONTROL = 0x00020000;
public const UInt32 WRITE_DAC = 0x00040000;
public const UInt32 WRITE_OWNER = 0x00080000;
public const UInt32 SYNCHRONIZE = 0x00100000;
public const UInt32 STANDARD_RIGHTS_ALL = (
READ_CONTROL |
WRITE_OWNER |
WRITE_DAC |
DELETE |
SYNCHRONIZE
);
public const UInt32 STANDARD_RIGHTS_REQUIRED = 0x000F0000u;
public const UInt32 STANDARD_RIGHTS_READ = 0x00020000u;
public const UInt32 SE_PRIVILEGE_ENABLED_BY_DEFAULT = 0x00000001u;
public const UInt32 SE_PRIVILEGE_ENABLED = 0x00000002u;
public const UInt32 SE_PRIVILEGE_REMOVED = 0x00000004u;
public const UInt32 SE_PRIVILEGE_USED_FOR_ACCESS = 0x80000000u;
public const UInt32 TOKEN_QUERY = 0x00000008;
public const UInt32 TOKEN_ADJUST_PRIVILEGES = 0x00000020;
public const UInt32 TOKEN_ASSIGN_PRIMARY = 0x00000001u;
public const UInt32 TOKEN_DUPLICATE = 0x00000002u;
public const UInt32 TOKEN_IMPERSONATE = 0x00000004u;
public const UInt32 TOKEN_QUERY_SOURCE = 0x00000010u;
public const UInt32 TOKEN_ADJUST_GROUPS = 0x00000040u;
public const UInt32 TOKEN_ADJUST_DEFAULT = 0x00000080u;
public const UInt32 TOKEN_ADJUST_SESSIONID = 0x00000100u;
public const UInt32 TOKEN_READ = (
STANDARD_RIGHTS_READ |
TOKEN_QUERY
);
public const UInt32 TOKEN_ALL_ACCESS = (
STANDARD_RIGHTS_REQUIRED |
TOKEN_ASSIGN_PRIMARY |
TOKEN_DUPLICATE |
TOKEN_IMPERSONATE |
TOKEN_QUERY |
TOKEN_QUERY_SOURCE |
TOKEN_ADJUST_PRIVILEGES |
TOKEN_ADJUST_GROUPS |
TOKEN_ADJUST_DEFAULT |
TOKEN_ADJUST_SESSIONID
);
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentProcess();
[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentThread();
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool LookupPrivilegeValue(string lpSystemName, string lpName, out LUID lpLuid);
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool AdjustTokenPrivileges(IntPtr TokenHandle, bool DisableAllPrivileges, ref TOKEN_PRIVILEGES NewState, UInt32 BufferLengthInBytes, IntPtr PreviousStateNull, IntPtr ReturnLengthInBytesNull);
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenProcessToken(IntPtr ProcessHandle, UInt32 DesiredAccess, out IntPtr TokenHandle);
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenThreadToken(IntPtr ThreadHandle, UInt32 DesiredAccess, bool OpenAsSelf, out IntPtr TokenHandle);
[DllImport("ntdll.dll", EntryPoint = "RtlAdjustPrivilege")]
public static extern int RtlAdjustPrivilege(
UInt32 Privilege,
bool Enable,
bool CurrentThread,
ref bool Enabled
);
[DllImport("Kernel32.dll", SetLastError = true)]
public static extern bool CloseHandle(IntPtr handle);
//
//
private static LUID LookupPrivilege(string privilegeName)
{
LUID privilegeValue = new LUID();
bool res = LookupPrivilegeValue(null, privilegeName, out privilegeValue);
if (!res)
{
throw new Exception("Error: LookupPrivilegeValue()");
}
return privilegeValue;
}
//
//
public static void AdjustPrivilege(string privilegeName, bool enable)
{
IntPtr accessToken = IntPtr.Zero;
bool res = false;
try
{
LUID privilegeValue = LookupPrivilege(privilegeName);
res = OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, false, out accessToken);
if (!res)
{
res = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, out accessToken);
if (!res)
{
throw new Exception("Error: OpenProcessToken()");
}
}
TOKEN_PRIVILEGES tokenPrivileges = new TOKEN_PRIVILEGES();
tokenPrivileges.PrivilegeCount = 1;
tokenPrivileges.Luid = privilegeValue;
if (enable)
{
tokenPrivileges.Attributes = SE_PRIVILEGE_ENABLED;
}
else
{
tokenPrivileges.Attributes = 0;
}
res = AdjustTokenPrivileges(accessToken, false, ref tokenPrivileges, (uint)System.Runtime.InteropServices.Marshal.SizeOf(tokenPrivileges), IntPtr.Zero, IntPtr.Zero);
if (!res)
{
throw new Exception("Error: AdjustTokenPrivileges()");
}
}
finally
{
if (accessToken != IntPtr.Zero)
{
CloseHandle(accessToken);
accessToken = IntPtr.Zero;
}
}
}
}
}
'@
if ([object]::Equals(('SystemPrivilege.Win32API.Privileges' -as [type]), $null)) {
Add-Type -TypeDefinition $win32api
}
}
includeSystemPrivileges;
$privileges|%{[SystemPrivilege.Win32API.Privileges]::AdjustPrivilege($_, $true)}
# Validation
whoami /priv|?{$_ -match "SeBackupPrivilege|SeRestorePrivilege"}
}
addSystemPrivilege;
################################## Add System Backup Privileges ####################################
################################## Commence File Copy Operations ####################################
function validatePaths{
param($inputData=$arr)
$thisClusterName=get-wmiobject -class "MSCluster_Cluster" -namespace "root\mscluster" -ErrorAction SilentlyContinue|select -ExpandProperty Name;
$objectLength=$inputData.length;
$castedArrayList=[System.Collections.ArrayList]$inputData;
for ($i=0;$i -lt $objectLength; $i++){
#$from=$inputData[$i].From;
#$referenceCluster=$inputData[$i].Clustername;
$from=$castedArrayList[$i].From;
$referenceCluster=$castedArrayList[$i].Clustername;
if(!($thisClusterName -like $referenceCluster) -OR !(test-path $from)){
write-host "Removing $inputData[$i]..."
# Remove row if any path doesn't resolve. Overcome limitations of Powershell's immutable array "fixed size" using this workaround
#$castedArrayList=[System.Collections.ArrayList]$inputData;
$castedArrayList.RemoveAt($i);
#$inputData=[Array]$castedArrayList; #reverse the casting and reassign to original Array
$objectLength--;
$i--;
} #else{write-host "$to is reachable."}
}
$inputData=[Array]$castedArrayList; #reverse the casting and reassign to original Array
return $inputData
}
[Array]$arr=validatePaths -inputData $arr;
# function to obtain the version of a particular excecutable
function getExecutableVersion{
param(
[string]$computername=$env:computername,
[string]$executablename="robocopy.exe"
)
$localComputerName=$env:computername
$isLocal=if($computername -match "(localhost|127.0.0.1|$localComputerName)"){$true}else{$false}
if ($isLocal){
$exeInfo=(get-item (get-command $executablename).Definition).versionInfo;
}else{
$exeInfo=invoke-command -computername $computername -scriptblock{
param($executable)
$executableVersion=(get-item (get-command $executable).Definition).versionInfo;
return $executableVersion;} -args $executablename
}
$exeVersion=$exeInfo.ProductVersion;
$exeLocation=$exeInfo.FileName;
$fileVersion=$exeInfo.FileVersion;
return "$executablename`: release $exeVersion $(if($exeVersion -ne $fileVersion){"| version '$fileVersion'"}) | location $exeLocation"
}
New-Item -ItemType Directory -Force -Path $logpath;
"Executing copying tasks..."
function createShadow{
[cmdletbinding()]
param(
[string]$targetVolume="C:\"
)
# Sanitation
if (!($targetVolume -like "*\")){$targetVolume+="\"}
$GLOBAL:shadowMount="C:\shadowcopy";
if(Test-Path $shadowMount){(Get-Item $shadowMount).Delete()}
write-host "Initiating VSS snapshot..."
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy"
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible")
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID }
$thisShadowPath = $thisShadow.DeviceObject + "\"
# Creating symlink
$voidOutput=cd C:
$voidOutput=cmd /c mklink /d $shadowMount $thisShadowPath | Out-Null
write-host "Shadow of $targetVolume has been made and it's accessible at this local file system (LFS): $shadowMount."
# Validation
if(Test-Path $shadowMount){
$snapshotId=$thisShadow.ID;
write-host "Snapshot $snapshotId has been created.";
return $snapshotId;
}else{
write-host "Failed to create client accessible VSS Snapshot.";
return $false;
}
}
function deleteShadow{
[cmdletbinding()]
param(
[string]$targetVolume="C:\",
[string]$lastSnapshotId
)
# Deterministic method of obtaining newest snapshot ID if it is not specified
if(!($lastSnapshotId)){
$lastSnapshotIdString=vssadmin list shadows /for=$targetVolume|`
%{if($_ -like "*Shadow Copy ID:*"){$_}}|`
select-object -last 1|`
%{[void]($_ -match "{(.*)}$"); $matches[1]}
$lastSnapshotId="{$lastSnapshotIdString}"
}
# Remove a single snapshot
write-host "Removing snapshot Id $lastSnapshotId..."
$voidOutput=cmd.exe /c vssadmin delete shadows /Shadow=$lastSnapshotId /quiet
# Remove symlink
write-host "Removing symlink $shadowMount..."
(Get-Item $shadowMount).Delete()
# Remove all Snapshots
#Get-WmiObject Win32_ShadowCopy | % {$_.delete()}
#vssadmin delete shadows /For=$targetVolume /Quiet
# Validation
#vssadmin list shadows /for=$targetVolume
$validateLastSnapshot=vssadmin list shadows /for=$targetVolume|`
%{if($_ -like "*Shadow Copy ID:*"){$_}}|`
select-object -last 1|`
%{[void]($_ -match "{(.*)}$"); $matches[1]}
$validateLastSnapshotId="{$validateLastSnapshot}"
if(!($validateLastSnapshotId -eq $lastSnapshotId)){
write-host "Last snapshot Id is now $validateLastSnapshotId"
return $true
}else{
write-host "$lastSnapshotId still exists. There was an error in its removal";
return $false
}
}
function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}
function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}
function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}
# This function is deprecated as it does not account for spaces in paths properly
function validateSourceAndDestination($thisblock){
$spacesCount=($thisblock.Split(' ')).Count-1
if ($spacesCount -eq 1){
$GLOBAL:source,$GLOBAL:destination=$thisblock.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $thisblock;
return $False;
}
}
function translateSource{
param(
[string]$uncPath
)
$uri = new-object System.Uri($uncPath)
$thisLocalPath=$uri.LocalPath
#$thisHost=$uri.Host
$GLOBAL:sourceVolume="$((Get-Item $thisLocalPath).PSDrive.Name)`:"
$translatedSource=$thisLocalPath -replace "$sourceVolume", $shadowMount
return $translatedSource
}
function checkDiskFree($volume){
<#
Excerpt from https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/ee692290(v=ws.10)?redirectedfrom=MSDN:
"For volumes less than 500 megabytes, the minimum is 50 megabytes of free space.
For volumes more than 500 megabytes, the minimum is 320 megabytes of free space.
It is recommended that least 1 gigabyte of free disk space on each volume if the volume size is more than 1 gigabyte."
#>
# Import variables
$thisNode="localhost"
$thisVolume=$volume
# Obtain disk information
$diskObject = Get-WmiObject Win32_LogicalDisk -ComputerName $thisNode -Filter "DeviceID='$thisVolume'"
$diskFree=[Math]::Round($diskObject.FreeSpace / 1MB)
$diskSize=[Math]::Round($diskObject.Size / 1MB)
switch ($diskSize){
{$diskSize -ge 1024} {if ($diskFree -gt 1024){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -ge 500} {if ($diskFree -gt 320){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -lt 500} {if ($diskFree -gt 50){$feasible=$True;}else{$feasible=$False;};break;}
}
return $feasible
}
function expandZipfile($file, $destination){
$shell = new-object -com shell.application
$zip = $shell.NameSpace($file)
foreach($item in $zip.items()){
$shell.Namespace($destination).copyhere($item)
}
}
function installEmcopy{
$emcopyIsInstalled=(Get-Command emcopy.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
if (!($emcopyIsInstalled)){
$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$emCopySource = "https://blog.kimconnect.com/wp-content/uploads/2019/08/emcopy.zip";
$destinationFile = "$tempDir\emcopy.zip";
try{[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12}catch{}
New-Item -ItemType Directory -Force -Path $tempDir
New-Item -ItemType Directory -Force -Path $extractionDir
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($emCopySource,$destinationFile);
expandZipfile $destinationFile -Destination $extractionDir
}else{
"EMCOPY is currently available in this system.`n";
}
}
function setACL{
get-acl -Path $source | Set-Acl -Path $destination # This command requires PowerShell 5
}
function Get-STFolderSize {
<#
.SYNOPSIS
Gets folder sizes using COM and by default with a fallback to robocopy.exe, with the
logging only option, which makes it not actually copy or move files, but just list them, and
the end summary result is parsed to extract the relevant data.
.DESCRIPTION
There is a -ComOnly parameter for using only COM, and a -RoboOnly parameter for using only
robocopy.exe with the logging only option.
The robocopy output also gives a count of files and folders, unlike the COM method output.
The default number of threads used by robocopy is 8, but I set it to 16 since this cut the
run time down to almost half in some cases during my testing. You can specify a number of
threads between 1-128 with the parameter -RoboThreadCount.
Both of these approaches are apparently much faster than .NET and Get-ChildItem in PowerShell.
The properties of the objects will be different based on which method is used, but
the "TotalBytes" property is always populated if the directory size was successfully
retrieved. Otherwise you should get a warning (and the sizes will be zero).
Online documentation: https://www.powershelladmin.com/wiki/Get_Folder_Size_with_PowerShell,_Blazingly_Fast.php
MIT license. https://opensource.org/license/MIT
Copyright (C) 2015-present, Joakim Borger Svendsen
All rights reserved.
Svendsen Tech.
.PARAMETER Path
Path or paths to measure size of.
.PARAMETER LiteralPath
Path or paths to measure size of, supporting wildcard characters
in the names, as with Get-ChildItem.
.PARAMETER Precision
Number of digits after decimal point in rounded numbers.
.PARAMETER RoboOnly
Do not use COM, only robocopy, for always getting full details.
.PARAMETER ComOnly
Never fall back to robocopy, only use COM.
.PARAMETER RoboThreadCount
Number of threads used when falling back to robocopy, or with -RoboOnly.
Default: 16 (gave the fastest results during my testing).
.PARAMETER ExcludeDirectory
Names and paths for directories to exclude, as supported by the RoboCopy.exe /XD syntax.
To guarantee its use, you need to use the parameter -RoboOnly. If COM is used, nothing
is filtered out.
.PARAMETER ExcludeFile
Names/paths/wildcards for files to exclude, as supported by the RoboCopy.exe /XF syntax.
To guarantee its use, you need to use the parameter -RoboOnly. If COM is used, nothing
is filtered out.
.EXAMPLE
Import-Module .\Get-FolderSize.psm1
PS C:\> 'C:\Windows', 'E:\temp' | Get-STFolderSize
.EXAMPLE
Get-STFolderSize -Path Z:\Database -Precision 2
.EXAMPLE
Get-STFolderSize -Path Z:\Users\*\AppData\*\Something -RoboOnly -RoboThreadCount 64
.EXAMPLE
Get-STFolderSize -Path "Z:\Database[0-9][0-9]" -RoboOnly
.EXAMPLE
Get-STFolderSize -LiteralPath 'A:\Full[HD]FloppyMovies' -ComOnly
Supports wildcard characters in the path name with -LiteralPath.
.EXAMPLE
PS C:\temp> dir -dir | Get-STFolderSize | select Path, TotalMBytes |
Sort-Object -Descending totalmbytes
Path TotalMBytes
---- -----------
C:\temp\PowerShellGetOld 1.2047
C:\temp\git 0.6562
C:\temp\Docker 0.3583
C:\temp\MergeCsv 0.1574
C:\temp\testdir 0.0476
C:\temp\DotNetVersionLister 0.0474
C:\temp\temp2 0.0420
C:\temp\WriteAscii 0.0328
C:\temp\tempbenchmark 0.0257
C:\temp\From PS Gallery Benchmark 0.0253
C:\temp\RemoveOldFiles 0.0238
C:\temp\modules 0.0234
C:\temp\STDockerPs 0.0216
C:\temp\RandomData 0.0205
C:\temp\GetSTFolderSize 0.0198
C:\temp\Benchmark 0.0151
.LINK
https://www.powershelladmin.com/wiki/Get_Folder_Size_with_PowerShell,_Blazingly_Fast.php
#>
[CmdletBinding(DefaultParameterSetName = "Path")]
param(
[Parameter(ParameterSetName = "Path",
Mandatory = $True,
ValueFromPipeline = $True,
ValueFromPipelineByPropertyName = $True,
Position = 0)]
[Alias('Name', 'FullName')]
[String[]] $Path,
[System.Int32] $Precision = 4,
[Switch] $RoboOnly,
[Switch] $ComOnly,
[Parameter(ParameterSetName = "LiteralPath",
Mandatory = $true,
Position = 0)] [String[]] $LiteralPath,
[ValidateRange(1, 128)] [Byte] $RoboThreadCount = 16,
[String[]] $ExcludeDirectory = @(),
[String[]] $ExcludeFile = @())
Begin {
$DefaultProperties = 'Path', 'TotalBytes', 'TotalMBytes', 'TotalGBytes', 'TotalTBytes', 'DirCount', 'FileCount',
'DirFailed', 'FileFailed', 'TimeElapsed', 'StartedTime', 'EndedTime'
if (($ExcludeDirectory.Count -gt 0 -or $ExcludeFile.Count -gt 0) -and -not $ComOnly) {
$DefaultProperties += @("CopiedDirCount", "CopiedFileCount", "CopiedBytes", "SkippedDirCount", "SkippedFileCount", "SkippedBytes")
}
if ($RoboOnly -and $ComOnly) {
Write-Error -Message "You can't use both -ComOnly and -RoboOnly. Default is COM with a fallback to robocopy." -ErrorAction Stop
}
if (-not $RoboOnly) {
$FSO = New-Object -ComObject Scripting.FileSystemObject -ErrorAction Stop
}
function Get-RoboFolderSizeInternal {
[CmdletBinding()]
param(
# Paths to report size, file count, dir count, etc. for.
[String[]] $Path,
[System.Int32] $Precision = 4)
begin {
if (-not (Get-Command -Name robocopy -ErrorAction SilentlyContinue)) {
Write-Warning -Message "Fallback to robocopy failed because robocopy.exe could not be found. Path '$p'. $([datetime]::Now)."
return
}
}
process {
foreach ($p in $Path) {
Write-Verbose -Message "Processing path '$p' with Get-RoboFolderSizeInternal. $([datetime]::Now)."
$RoboCopyArgs = @("/L","/S","/NJH","/BYTES","/FP","/NC","/NDL","/TS","/XJ","/R:0","/W:0","/MT:$RoboThreadCount")
if ($ExcludeDirectory.Count -gt 0) {
$RoboCopyArgs += @(@("/XD") + @($ExcludeDirectory))
}
if ($ExcludeFile.Count -gt 0) {
$RoboCopyArgs += @(@("/XF") + @($ExcludeFile))
}
[DateTime] $StartedTime = [DateTime]::Now
[String] $Summary = robocopy $p NULL $RoboCopyArgs | Select-Object -Last 8
[DateTime] $EndedTime = [DateTime]::Now
#[String] $DefaultIgnored = '(?:\s+[\-\d]+){3}'
[Regex] $HeaderRegex = '\s+Total\s*Copied\s+Skipped\s+Mismatch\s+FAILED\s+Extras'
[Regex] $DirLineRegex = "Dirs\s*:\s*(?<DirCount>\d+)(?<CopiedDirCount>\s+[\-\d]+)(?<SkippedDirCount>\s+[\-\d]+)(?:\s+[\-\d]+)\s+(?<DirFailed>\d+)\s+[\-\d]+"
[Regex] $FileLineRegex = "Files\s*:\s*(?<FileCount>\d+)(?<CopiedFileCount>\s+[\-\d]+)(?<SkippedFileCount>\s+[\-\d]+)(?:\s+[\-\d]+)\s+(?<FileFailed>\d+)\s+[\-\d]+"
[Regex] $BytesLineRegex = "Bytes\s*:\s*(?<ByteCount>\d+)(?<CopiedBytes>\s+[\-\d]+)(?<SkippedBytes>\s+[\-\d]+)(?:\s+[\-\d]+)\s+(?<BytesFailed>\d+)\s+[\-\d]+"
[Regex] $TimeLineRegex = "Times\s*:\s*.*"
[Regex] $EndedLineRegex = "Ended\s*:\s*(?<EndedTime>.+)"
if ($Summary -match "$HeaderRegex\s+$DirLineRegex\s+$FileLineRegex\s+$BytesLineRegex\s+$TimeLineRegex\s+$EndedLineRegex") {
New-Object PSObject -Property @{
Path = $p
TotalBytes = [Decimal] $Matches['ByteCount']
TotalMBytes = [Math]::Round(([Decimal] $Matches['ByteCount'] / 1MB), $Precision)
TotalGBytes = [Math]::Round(([Decimal] $Matches['ByteCount'] / 1GB), $Precision)
TotalTBytes = [Math]::Round(([Decimal] $Matches['ByteCount'] / 1TB), $Precision)
BytesFailed = [Decimal] $Matches['BytesFailed']
DirCount = [Decimal] $Matches['DirCount']
FileCount = [Decimal] $Matches['FileCount']
DirFailed = [Decimal] $Matches['DirFailed']
FileFailed = [Decimal] $Matches['FileFailed']
TimeElapsed = [Math]::Round([Decimal] ($EndedTime - $StartedTime).TotalSeconds, $Precision)
StartedTime = $StartedTime
EndedTime = $EndedTime
CopiedDirCount = [Decimal] $Matches['CopiedDirCount']
CopiedFileCount = [Decimal] $Matches['CopiedFileCount']
CopiedBytes = [Decimal] $Matches['CopiedBytes']
SkippedDirCount = [Decimal] $Matches['SkippedDirCount']
SkippedFileCount = [Decimal] $Matches['SkippedFileCount']
SkippedBytes = [Decimal] $Matches['SkippedBytes']
} | Select-Object -Property $DefaultProperties
}
else {
Write-Warning -Message "Path '$p' output from robocopy was not in an expected format."
}
}
}
}
}
Process {
if ($PSCmdlet.ParameterSetName -eq "Path") {
$Paths = @(Resolve-Path -Path $Path | Select-Object -ExpandProperty ProviderPath -ErrorAction SilentlyContinue)
}
else {
$Paths = @(Get-Item -LiteralPath $LiteralPath | Select-Object -ExpandProperty FullName -ErrorAction SilentlyContinue)
}
foreach ($p in $Paths) {
Write-Verbose -Message "Processing path '$p'. $([DateTime]::Now)."
if (-not (Test-Path -LiteralPath $p -PathType Container)) {
Write-Warning -Message "$p does not exist or is a file and not a directory. Skipping."
continue
}
# We know we can't have -ComOnly here if we have -RoboOnly.
if ($RoboOnly) {
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
$ErrorActionPreference = 'Stop'
try {
$StartFSOTime = [DateTime]::Now
$TotalBytes = $FSO.GetFolder($p).Size
$EndFSOTime = [DateTime]::Now
if ($null -eq $TotalBytes) {
if (-not $ComOnly) {
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
else {
Write-Warning -Message "Failed to retrieve folder size for path '$p': $($Error[0].Exception.Message)."
}
}
}
catch {
if ($_.Exception.Message -like '*PERMISSION*DENIED*') {
if (-not $ComOnly) {
Write-Verbose "Caught a permission denied. Trying robocopy."
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
else {
Write-Warning "Failed to process path '$p' due to a permission denied error: $($_.Exception.Message)"
}
}
Write-Warning -Message "Encountered an error while processing path '$p': $($_.Exception.Message)"
continue
}
$ErrorActionPreference = 'Continue'
New-Object PSObject -Property @{
Path = $p
TotalBytes = [Decimal] $TotalBytes
TotalMBytes = [Math]::Round(([Decimal] $TotalBytes / 1MB), $Precision)
TotalGBytes = [Math]::Round(([Decimal] $TotalBytes / 1GB), $Precision)
TotalTBytes = [Math]::Round(([Decimal] $TotalBytes / 1TB), $Precision)
BytesFailed = $null
DirCount = $null
FileCount = $null
DirFailed = $null
FileFailed = $null
TimeElapsed = [Math]::Round(([Decimal] ($EndFSOTime - $StartFSOTime).TotalSeconds), $Precision)
StartedTime = $StartFSOTime
EndedTime = $EndFSOTime
} | Select-Object -Property $DefaultProperties
}
}
End {
if (-not $RoboOnly) {
[Void] [System.Runtime.Interopservices.Marshal]::ReleaseComObject($FSO)
}
[System.GC]::Collect()
#[System.GC]::WaitForPendingFinalizers()
}
}
<# Source: https://www.powershelladmin.com/wiki/Get_Folder_Size_with_PowerShell,_Blazingly_Fast.php
.SYNOPSIS
Gets folder sizes using COM and by default with a fallback to robocopy.exe, with the
logging only option, which makes it not actually copy or move files, but just list them, and
the end summary result is parsed to extract the relevant data.
There is a -ComOnly parameter for using only COM, and a -RoboOnly parameter for using only
robocopy.exe with the logging only option.
The robocopy output also gives a count of files and folders, unlike the COM method output.
The default number of threads used by robocopy is 8, but I set it to 16 since this cut the
run time down to almost half in some cases during my testing. You can specify a number of
threads between 1-128 with the parameter -RoboThreadCount.
Both of these approaches are apparently much faster than .NET and Get-ChildItem in PowerShell.
The properties of the objects will be different based on which method is used, but
the "TotalBytes" property is always populated if the directory size was successfully
retrieved. Otherwise you should get a warning (and the sizes will be zero).
Online documentation: https://www.powershelladmin.com/wiki/Get_Folder_Size_with_PowerShell,_Blazingly_Fast.php
MIT license. https://opensource.org/license/MIT
Copyright (C) 2015-2017, Joakim Svendsen
All rights reserved.
Svendsen Tech.
.PARAMETER Path
Path or paths to measure size of.
.PARAMETER LiteralPath
Path or paths to measure size of, supporting wildcard characters
in the names, as with Get-ChildItem.
.PARAMETER Precision
Number of digits after decimal point in rounded numbers.
.PARAMETER RoboOnly
Do not use COM, only robocopy, for always getting full details.
.PARAMETER ComOnly
Never fall back to robocopy, only use COM.
.PARAMETER RoboThreadCount
Number of threads used when falling back to robocopy, or with -RoboOnly.
Default: 16 (gave the fastest results during my testing).
.EXAMPLE
. .\Get-FolderSize.ps1
PS C:\> 'C:\Windows', 'E:\temp' | Get-FolderSize
.EXAMPLE
Get-FolderSize -Path Z:\Database -Precision 2
.EXAMPLE
Get-FolderSize -Path Z:\Database -RoboOnly -RoboThreadCount 64
.EXAMPLE
Get-FolderSize -Path Z:\Database -RoboOnly
.EXAMPLE
Get-FolderSize A:\FullHDFloppyMovies -ComOnly
#>
function Get-FolderSize {
[CmdletBinding(DefaultParameterSetName = "Path")]
param(
[Parameter(ParameterSetName = "Path",
Mandatory = $true,
ValueFromPipeline = $true,
ValueFromPipelineByPropertyName = $true,
Position = 0)]
[Alias('Name', 'FullName')]
[string[]] $Path,
[int] $Precision = 4,
[switch] $RoboOnly,
[switch] $ComOnly,
[Parameter(ParameterSetName = "LiteralPath",
Mandatory = $true,
Position = 0)] [string[]] $LiteralPath,
[ValidateRange(1, 128)] [byte] $RoboThreadCount = 16)
begin {
if ($RoboOnly -and $ComOnly) {
Write-Error -Message "You can't use both -ComOnly and -RoboOnly. Default is COM with a fallback to robocopy." -ErrorAction Stop
}
if (-not $RoboOnly) {
$FSO = New-Object -ComObject Scripting.FileSystemObject -ErrorAction Stop
}
function Get-RoboFolderSizeInternal {
[CmdletBinding()]
param(
# Paths to report size, file count, dir count, etc. for.
[string[]] $Path,
[int] $Precision = 4)
begin {
if (-not (Get-Command -Name robocopy -ErrorAction SilentlyContinue)) {
Write-Warning -Message "Fallback to robocopy failed because robocopy.exe could not be found. Path '$p'. $([datetime]::Now)."
return
}
}
process {
foreach ($p in $Path) {
Write-Verbose -Message "Processing path '$p' with Get-RoboFolderSizeInternal. $([datetime]::Now)."
$RoboCopyArgs = @("/L","/S","/NJH","/BYTES","/FP","/NC","/NDL","/TS","/XJ","/R:0","/W:0","/MT:$RoboThreadCount")
[datetime] $StartedTime = [datetime]::Now
[string] $Summary = robocopy $p NULL $RoboCopyArgs | Select-Object -Last 8
[datetime] $EndedTime = [datetime]::Now
[regex] $HeaderRegex = '\s+Total\s*Copied\s+Skipped\s+Mismatch\s+FAILED\s+Extras'
[regex] $DirLineRegex = 'Dirs\s*:\s*(?<DirCount>\d+)(?:\s+\d+){3}\s+(?<DirFailed>\d+)\s+\d+'
[regex] $FileLineRegex = 'Files\s*:\s*(?<FileCount>\d+)(?:\s+\d+){3}\s+(?<FileFailed>\d+)\s+\d+'
[regex] $BytesLineRegex = 'Bytes\s*:\s*(?<ByteCount>\d+)(?:\s+\d+){3}\s+(?<BytesFailed>\d+)\s+\d+'
[regex] $TimeLineRegex = 'Times\s*:\s*(?<TimeElapsed>\d+).*'
[regex] $EndedLineRegex = 'Ended\s*:\s*(?<EndedTime>.+)'
if ($Summary -match "$HeaderRegex\s+$DirLineRegex\s+$FileLineRegex\s+$BytesLineRegex\s+$TimeLineRegex\s+$EndedLineRegex") {
New-Object PSObject -Property @{
Path = $p
TotalBytes = [decimal] $Matches['ByteCount']
TotalMBytes = [math]::Round(([decimal] $Matches['ByteCount'] / 1MB), $Precision)
TotalGBytes = [math]::Round(([decimal] $Matches['ByteCount'] / 1GB), $Precision)
BytesFailed = [decimal] $Matches['BytesFailed']
DirCount = [decimal] $Matches['DirCount']
FileCount = [decimal] $Matches['FileCount']
DirFailed = [decimal] $Matches['DirFailed']
FileFailed = [decimal] $Matches['FileFailed']
TimeElapsed = [math]::Round([decimal] ($EndedTime - $StartedTime).TotalSeconds, $Precision)
StartedTime = $StartedTime
EndedTime = $EndedTime
} | Select-Object -Property Path, TotalBytes, TotalMBytes, TotalGBytes, DirCount, FileCount, DirFailed, FileFailed, TimeElapsed, StartedTime, EndedTime
}
else {
Write-Warning -Message "Path '$p' output from robocopy was not in an expected format."
}
}
}
}
}
process {
if ($PSCmdlet.ParameterSetName -eq "Path") {
$Paths = @(Resolve-Path -Path $Path | Select-Object -ExpandProperty ProviderPath -ErrorAction SilentlyContinue)
}
else {
$Paths = @(Get-Item -LiteralPath $LiteralPath | Select-Object -ExpandProperty FullName -ErrorAction SilentlyContinue)
}
foreach ($p in $Paths) {
Write-Verbose -Message "Processing path '$p'. $([datetime]::Now)."
if (-not (Test-Path -LiteralPath $p -PathType Container)) {
Write-Warning -Message "$p does not exist or is a file and not a directory. Skipping."
continue
}
# We know we can't have -ComOnly here if we have -RoboOnly.
if ($RoboOnly) {
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
$ErrorActionPreference = 'Stop'
try {
$StartFSOTime = [datetime]::Now
$TotalBytes = $FSO.GetFolder($p).Size
$EndFSOTime = [datetime]::Now
if ($null -eq $TotalBytes) {
if (-not $ComOnly) {
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
else {
Write-Warning -Message "Failed to retrieve folder size for path '$p': $($Error[0].Exception.Message)."
}
}
}
catch {
if ($_.Exception.Message -like '*PERMISSION*DENIED*') {
if (-not $ComOnly) {
Write-Verbose "Caught a permission denied. Trying robocopy."
Get-RoboFolderSizeInternal -Path $p -Precision $Precision
continue
}
else {
Write-Warning "Failed to process path '$p' due to a permission denied error: $($_.Exception.Message)"
}
}
Write-Warning -Message "Encountered an error while processing path '$p': $($_.Exception.Message)"
continue
}
$ErrorActionPreference = 'Continue'
New-Object PSObject -Property @{
Path = $p
TotalBytes = [decimal] $TotalBytes
TotalMBytes = [math]::Round(([decimal] $TotalBytes / 1MB), $Precision)
TotalGBytes = [math]::Round(([decimal] $TotalBytes / 1GB), $Precision)
BytesFailed = $null
DirCount = $null
FileCount = $null
DirFailed = $null
FileFailed = $null
TimeElapsed = [math]::Round(([decimal] ($EndFSOTime - $StartFSOTime).TotalSeconds), $Precision)
StartedTime = $StartFSOTime
EndedTime = $EndFSOTime
} | Select-Object -Property Path, TotalBytes, TotalMBytes, TotalGBytes, DirCount, FileCount, DirFailed, FileFailed, TimeElapsed, StartedTime, EndedTime
}
}
end {
if (-not $RoboOnly) {
[void][System.Runtime.Interopservices.Marshal]::ReleaseComObject($FSO)
}
[gc]::Collect()
[gc]::WaitForPendingFinalizers()
}
}
function compareDirectories{
param(
$source,
$destination,
$log,
$sampleSize
)
if ($powerShellVersion -lt 4){
# Slower method of obtaining directory stats for older PowerShell versions
$sourceItems=Get-ChildItem $source -Recurse -Force
$sourceFiles=$sourceItems|?{!$_.PSisContainer}
$sourceFilesCount=$sourceFiles.count
if(!($sourceFilesCount)){$sourceFilesCount=0}
$sourceFoldersCount=$sourceItems.count - $sourceFilesCount
if(!($sourceFoldersCount)){$sourceFoldersCount=0}
if($calculateSizeOnDisk){
$sourceStorage=(getSizeOnDisk -path $source) / 1GB;
# Alternative method of calculating storage
#$storage=(Get-ChildItem $source -Recurse -Force | where { !$_.PSisContainer } | %{getFileSizeOnDisk $_.FullName}).Sum
}else{
$sourceStorage=($sourceFiles | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;
}
$destinationItems=Get-ChildItem $destination -Recurse -Force
$destinationFiles=$destinationItems|?{!$_.PSisContainer}
$destinationFilesCount=$destinationFiles.count
if(!($destinationFilesCount)){$destinationFilesCount=0}
$destinationFoldersCount=$destinationItems.count - $destinationFilesCount
if(!($destinationFoldersCount)){$destinationFoldersCount=0}
if($calculateSizeOnDisk){
$destinationStorage=(getSizeOnDisk -path $destination) / 1GB;
# Alternative method of calculating storage
#$storage=(Get-ChildItem $destination -Recurse -Force | where { !$_.PSisContainer } | %{getFileSizeOnDisk $_.FullName}).Sum
}else{
$destinationStorage=($destinationFiles | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;
}
}else{
# Faster method - doesn't work for PowerShell 2.0
$sourceStats=Get-STFolderSize -Path $source -RoboOnly
$sourceFilesCount=$sourceStats.FileCount
$sourceFoldersCount=$sourceStats.DirCount
$sourceStorage=$sourceStats.TotalGBytes
$destinationStats=Get-STFolderSize -Path $destination -RoboOnly
$destinationFilesCount=$destinationStats.FileCount
$destinationFoldersCount=$destinationStats.DirCount
$destinationStorage=$destinationStats.TotalGBytes
}
if($checkFileHashes){
if(!($sourceItems)){$sourceItems=Get-ChildItem $source -Recurse -Force}
if(!($destinationItems)){$destinationItems=Get-ChildItem $destination -Recurse -Force}
$sourceHashes=$sourceItems|% {Get-FileHash –Path $_.FullName}
$destinationHashes=$destinationItems|% {Get-FileHash –Path $_.FullName}
$fileDifferences=(Compare-Object -ReferenceObject $sourceHashes -DifferenceObject $destinationHashes -Property hash -PassThru).Path|Out-String
if($fileDifferences){
$directoryStats+="`r`n---------------------------File Differences:---------------------------`r`n$fileDifferences";
}else{
$directoryStats+="`r`n---------------------------File Differences:---------------------------`r`nNONE";
}
}
# Output files, folder counts, file differences, and directory stats
$directoryStats="`r`n---------------------------Source: $sourceFilesCount files, $sourceFoldersCount folders VS Destination: $destinationFilesCount files, $destinationFoldersCount folders---------------------------";
$percentDifference=[math]::Round(((($destinationStorage-$sourceStorage)/$sourceStorage)*100),2)
if($percentDifference -gt 0){
$gainLoss="$percentDifference`% more storage utilized at destination";
}else{
if($percentDifference -eq 0){
$gainLoss="source and destination storage stats are matching";
}else{
$gainLoss="$([math]::abs($percentDifference))`% less storage utilized at destination";
}
}
$directoryStats+="`r`n---------------------------$gainLoss`: Source $([math]::Round($sourceStorage,2)) GiB vs Destination $([math]::Round($destinationStorage,2)) GiB---------------------------";
write-host $directoryStats;
Add-Content $log $directoryStats;
if($toSampleTimeStamp){
$timeStampTimer=[System.Diagnostics.Stopwatch]::StartNew()
# Enable Remote to remote symlink following if it's not already set
$r2rEnabled=fsutil behavior query SymlinkEvaluation | select-string -Pattern "Remote to remote symbolic links are enabled."
if (!($r2rEnabled)){$nullOutput=fsutil behavior set SymlinkEvaluation R2R:1;}
$sampleSourceFiles=$sourceFiles|?{$_.FullName.Length -lt 260}|get-random -count $sampleSize | % {$_.FullName}
$filesCount=$sampleSourceFiles.Count
$rootCount=$source.Length
$commonDenominatingPaths=$sampleSourceFiles|%{$_.substring($rootCount,$_.length-$rootCount)}
$sampleDestinationFiles=$commonDenominatingPaths | %{if([string]$_[0] -eq "\"){[string]$destination+[string]$_}else{[string]$destination+"\"+[string]$_};}
$badStamps=0;
if ($filesCount -gt 1){
write-host "Checking a sample of $sampleSize for any time stamp inaccuracies..."
for ($i=0;$i -lt $sampleSourceFiles.Length;$i++){
$sourceFile=$sampleSourceFiles[$i];
$destinationFile=$sampleDestinationFiles[$i];
$sourceTimeStamp=(gi "$sourceFile").LastWriteTime;
$destinationTimeStamp=(gi "$destinationFile").LastWriteTime;
$dTimeIsNull=$destinationTimeStamp -eq $null
$sTimeIsNull=$sourceTimeStamp -eq $null
if ($dTimeIsNull -or $sTimeIsNull){
if ($sTimeIsNull){
$output+="`r`nUnable to obtain timestamp of $sourceFile";
}else{
$output+="`r`nUnable to obtain timestamp of $destinationFile";
}
$filesCount--;
}else{
if ($sourceTimeStamp -ne $destinationTimeStamp){
$output+="`r`n$destinationFile timestamp of $destinationTimeStamp DOES NOT MATCH its source $sourceFile timestamp of $sourceTimeStamp";
$badStamps++;
}
}
}
# Output
$timeStampElapsedHours=[math]::round($timeStampTimer.Elapsed.TotalHours,2);
$output="`r`n------------$((($filesCount-$badStamps)/$filesCount).tostring('P')) of the files in a sample of $filesCount are having accurate time stamps. Command has taken $timeStampElapsedHours hours--------------"+$output;
}else{
$output="`r`n----------------------Insufficient number of files to compare time stamps----------------------`r`n";
}
write-host $output;
Add-Content $log $output;
$timeStampTimer.Stop();
}
return $sourceStorage;
}
function triggerEmcopy{
param(
[string]$fromSource,
[string]$toDestination,
[string]$emswitches="/o /secforce /de /sd /c /r:0 /w:0 /th 64 /s /purge /sdd /stream"
)
if(!(Get-Command emcopy.exe -ea silentlycontinue)){installEmcopy};
write-host "Emcopy has started...";
$expression="emcopy.exe '$fromSource' '$toDestination' $emswitches /xd '$shadowMount\System Volume Information' '$shadowMount\`$RECYCLE.BIN'";
try{
write-host $expression;
Invoke-Expression $expression;
}
catch{
# Record any errors into log and continue to next item
continue;
}
write-host "`r`nEmcopy process has finished. Log is now generated at: $logFile";
}
function triggerRobocopy{
param(
[string]$fromSource,
[string]$toDestination,
[string]$roboSwitches="/TBD /FFT /NS /NC /S /E /COPY:DATS /DCOPY:T /PURGE /MIR /NP /XO /XJF /XJD /R:0 /W:0"
)
write-host "Robocopy has started...";
$fromSource="'$fromSource'"
$toDestination="'$toDestination'"
$expression="robocopy.exe $fromSource $toDestination $roboSwitches"
try{
write-host $expression;
#cmd /c pause;
Invoke-Expression $expression;
}catch{
# Record any errors into log and continue to next item
continue;
}
"`r`nEmcopy process has finished. Log is now generated at: $logFile";
}
Function startCopy{
param(
$GLOBAL:source=$from,
$GLOBAL:destination=$to,
$takeSnapshot,
$removeSnapshot
)
$GLOBAL:block="$from $to"
$totalSeconds=0;
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
if ($toValidateInputs){
$validatedInputs=validateSourceAndDestination $block
}else{
$validatedInputs=$true
}
if ($validatedInputs){
$translatedSource=translateSource -uncPath $source;
#$translatedBlock="$translatedSource $thisDestination";
#write-host "triggerRobocopy -source $translatedSource -destination $destination -roboSwitches $robocopySwitches;"
#cmd /c pause;
if (checkDiskFree $sourceVolume){
# Creating Shadow if it doesn't exist
if ($takeSnapshot){
$lastSnapshotId=createShadow $sourceVolume;
}else{
$lastSnapshotId="";
write-host "Not creating a duplicate shadow in this iteration.";
}
Try{
if($useEmCopy){
triggerEmcopy -fromSource $translatedSource -toDestination $destination -emswitches $emcopySwitches;
}else{
triggerRobocopy -fromSource $translatedSource -toDestination $destination -roboSwitches $robocopySwitches;
}
}
catch{
write-host "$Error";
continue;
}
# Deleting snapshot if next drive letter is different
if ($removeSnapshot){
deleteShadow -targetVolume $sourceVolume -lastSnapshotId $lastSnapshotId;
}else{
Write-Host "Not removing snapshot $lastSnapshotId because next one is on the same drive letter.";
}
write-host "Log for this file copy 'pass' has been generated at $logFile";
}else{
$message="`r`nNot enough disk space to create a VSS snapshot.`r`nSkipping $block...`r`n";
write-host $message;
Add-Content $logFile $message
}
}
if($compareDirectories){
$referenceStorage=compareDirectories -source $source -destination $destination -log $logFile -sampleSize $sampleSize;
$GLOBAL:totalStorage+=$referenceStorage;
}else{
$referenceStorage=(Get-STFolderSize -Path $source).TotalGBytes
$GLOBAL:totalStorage+=$referenceStorage;
}
# Get time elapsed
$secondsElapsed=$stopWatch.Elapsed.TotalSeconds;
$GLOBAL:totalTime+=$secondsElapsed;
write-host $totalTime;
# Add total time display to log
$hours=[math]::round($($secondsElapsed/3600),2);
$speed=[math]::Round($referenceStorage/($secondsElapsed/3600),2);
$speedComment="at the speed of $speed GiB/hr";
Add-Content $logFile "`r`n-----------------------------------Time elapsed for this pass: $hours hours$(if($checkFiles){$speedComment})----------------------------------";
}
Function copyWithoutReport{
param(
$GLOBAL:source=$from,
$GLOBAL:destination=$to,
$takeSnapshot,
$removeSnapshot
)
$GLOBAL:block="$from $to"
$totalSeconds=0;
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
if ($toValidateInputs){
$validatedInputs=validateSourceAndDestination $block
}else{
$validatedInputs=$true
}
if ($validatedInputs){
$translatedSource=translateSource -uncPath $source;
#$translatedBlock="$translatedSource $thisDestination";
#write-host "triggerRobocopy -source $translatedSource -destination $destination -roboSwitches $robocopySwitches;"
#cmd /c pause;
if (checkDiskFree $sourceVolume){
# Creating Shadow if it doesn't exist
if ($takeSnapshot){
$lastSnapshotId=createShadow $sourceVolume;
}else{
$lastSnapshotId="";
write-host "Not creating a duplicate shadow in this iteration.";
}
Try{
if($useEmCopy){
triggerEmcopy -fromSource $translatedSource -toDestination $destination -emswitches $emcopySwitches;
}else{
triggerRobocopy -fromSource $translatedSource -toDestination $destination -roboSwitches $robocopySwitches;
}
}
catch{
write-host "$Error";
continue;
}
# Deleting snapshot if next drive letter is different
if ($removeSnapshot){
deleteShadow -targetVolume $sourceVolume -lastSnapshotId $lastSnapshotId;
}else{
Write-Host "Not removing snapshot $lastSnapshotId because next one is on the same drive letter.";
}
write-host "Log for this file copy 'pass' has been generated at $logFile";
}else{
$message="`r`nNot enough disk space to create a VSS snapshot.`r`nSkipping $block...`r`n";
write-host $message;
Add-Content $logFile $message
}
}
# Get time elapsed
$secondsElapsed=$stopWatch.Elapsed.TotalSeconds;
$GLOBAL:totalTime+=$secondsElapsed;
write-host $totalTime;
# Add total time display to log
$hours=[math]::round($($secondsElapsed/3600),2);
Add-Content $logFile "`r`n-----------------------------------Time elapsed for this pass: $hours hours----------------------------------";
}
function getSizeOnDisk{
param (
[string]$path='.'
)
add-type -assemblyName "System.ServiceProcess"
<#
Unable to find type [Win32.Disk]: make sure that the assembly containing this type is loaded.
At \\sherver007\filecopy.ps1:1171 char:25
+ $size = [Win32.Disk] <<<< ::GetSizeOnDisk($_.FullName)
+ CategoryInfo : InvalidOperation: (Win32.Disk:String) [], RuntimeException
+ FullyQualifiedErrorId : TypeNotFound
#>
$getSizeSource = @"
using System;
using System.Runtime.InteropServices;
using System.ComponentModel;
using System.IO;
namespace Win32
{
public class Disk {
[DllImport("kernel32.dll")]
static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
[Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);
public static ulong GetSizeOnDisk(string filename)
{
uint HighOrderSize;
uint LowOrderSize;
ulong size;
FileInfo file = new FileInfo(filename);
LowOrderSize = GetCompressedFileSizeW(file.FullName, out HighOrderSize);
if (HighOrderSize == 0 && LowOrderSize == 0xffffffff)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
else {
size = ((ulong)HighOrderSize << 32) + LowOrderSize;
return size;
}
}
}
}
"@
Add-Type -TypeDefinition $getSizeSource
$totalSize=0;
$count=0;
$driveLetter=(Get-Item $path).PSDrive.Root
$clusterSize=(Get-WmiObject -Class Win32_Volume | Where-Object {$_.Name -eq $driveLetter}).BlockSize
if(!($clusterSize)){$clusterSize=$clusterSizeOfNetworkShare}
Get-ChildItem $path -Recurse -Force | where { !$_.PSisContainer }|% {
$size = [Win32.Disk]::GetSizeOnDisk($_.FullName)
#$AFZ=[MidpointRounding]::AwayFromZero
#$roundedSize=[math]::Round($size/$ClusterSize+0.5,$AFZ)*$ClusterSize
if($size -gt $clusterSize){$roundedSize=[math]::ceiling($size/$clusterSize)*$clusterSize}else{$roundedSize=$clusterSize;}
$count=$count+1
#"$count`. $($_.FullName): $size vs $roundedSize"
$totalSize+=$roundedSize
}
#$foldersCount=(Get-ChildItem $path -Recurse -Force | where { $_.PSisContainer }).Count
#write-output "$count files $foldersCount folders total size: $totalSize"
return $totalSize;
}
function getFileSizeOnDisk{
param(
[string]$volumeName="C:\",
[int32]$clusterSize=4096,
[string]$file="C:\Windows\system32\cmd.exe"
)
if(!($clusterSize)){
$clusterSize=(Get-WmiObject -Class Win32_Volume | Where-Object {$_.Name -eq $volumeName}).BlockSize;
#write-host $clusterSize
}
$size=(get-item $file).Length
$remainder=$size%$clusterSize
#write-host $remainder
if ($remainder -eq 0){
$sizeOnDisk=$size
}else{
$sizeOnDisk=$size-$remainder+$clusterSize
}
return $sizeOnDisk
}
function generateErrorsLog{
$rawLog=Get-Content -Path $logFile;
$regexErrors="( : ERROR \()|( : WARNING : )"
$errorsFound="";
$rawLog | % {
$errorLine=$_ -match $regexErrors;
if ($errorLine){
$errorsFound+="$_`r`n"
}
}
if($errorsFound){
$errorsFound="`r`n===================================== ERRORS Section Begins ======================================`r`n"+$errorsFound+"===================================== ERRORS Section Ends ======================================`r`n"
Add-Content $logFile $errorsFound
}
}
function sendEmailViaRelay{
param($emailMessage,$log)
# Define sendmail variables
$from = "[email protected]"
$to = "'[email protected]','[email protected]'"
$cc = "'[email protected]','[email protected]'"
$subject = "$env:computername Final-Sync Message"
$body = $emailMessage
$smtpServer = "mail.kimconnect.net"
$logAttachments=if($log){" -Attachments '$log'"}else{""}
# This is the Windows' version of sendmail command
$sendmailCommand="Send-MailMessage -From $from -to $to -Cc $cc -Subject '$subject' -Body '$body'$logAttachments -SmtpServer $smtpServer -DeliveryNotificationOption OnSuccess"
Invoke-Expression $sendmailCommand
}
function stopAnyService{
param(
[string]$serviceName,
[string]$computerName=$env:computername
)
function executeKillCommand($serviceName){
write-host "including prerequisites..."
.{
# Prerequisite commands
$chocoAvailable="get-command choco -ErrorAction SilentlyContinue";
$psexecAvailable="get-command psexec -ErrorAction SilentlyContinue";
$setAclAvailable="get-command setacl -ErrorAction SilentlyContinue";
if (!(Invoke-Expression $chocoAvailable)) {
write-host "Installing Choco...";
Set-ExecutionPolicy Bypass -Scope Process -Force;
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'));
}
if (!(Invoke-Expression $chocoAvailable)) {
write-host "Unable to install Chocolatey automation tool. Program now aborts.";
break;
}
if(!(Invoke-Expression $psexecAvailable)){
$pendingRebootTests = @(
@{
Name = 'RebootPending'
Test = { Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing' -Name 'RebootPending' -ErrorAction SilentlyContinue }
TestType = 'ValueExists'
}
@{
Name = 'RebootRequired'
Test = { Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update' -Name 'RebootRequired' -ErrorAction SilentlyContinue }
TestType = 'ValueExists'
}
@{
Name = 'PendingFileRenameOperations'
Test = { Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager' -Name 'PendingFileRenameOperations' -ErrorAction SilentlyContinue }
TestType = 'NonNullValue'
}
)
foreach ($test in $pendingRebootTests) {
$pendingReboot=Invoke-Command -ScriptBlock $test.Test
if($pendingReboot){
write-host "$env:computername currently has a pending reboot requirement. Aborting session...";
break;
}
}
try {
$status = ([wmiclass]"\\.\root\ccm\clientsdk:CCM_ClientUtilities").DetermineIfRebootPending()
if(($status -ne $null) -and $status.RebootPending){
write-host "$env:computername currently has a pending reboot requirement. Aborting session...";
break;
}
}catch{}
write-host "Installing PSExec...";
choco install sysinternals -y -force;
}
if (!(Invoke-Expression $psexecAvailable)) {
write-host "Unable to install psexec. Program now aborts.";
break;
}
if(!(Invoke-Expression $setAclAvailable)){
write-host "Installing setacl...";
choco install setacl -y -force;
}
if (!(Invoke-Expression $setAclAvailable)) {
write-host "Unable to install setACL. Program now aborts.";
break;
}
write-host "Done.";
}
function forceKillService ($service){
if ($service.Status -ne "Stopped"){
# Try to stop the service as normal - only modify permissions and retry upon encountering errors
try{
Stop-Service $serviceName -Force -ErrorAction Stop;
}
catch{
$serviceRunas=(Get-WMIObject win32_service |?{$_ -like "*$serviceName*"}).StartName;
<# $serviceRunas=(Get-CIMInstance win32_service |?{$_ -like "*$serviceName*"}).StartName;
The term 'Get-CIMInstance' is not recognized as the name of a cmdlet, function, script file, or operable program. Check
the spelling of the name, or if a path was included, verify that the path is correct and try again.
At line:1 char:16
+ Get-CIMInstance <<<< win32_service
+ CategoryInfo : ObjectNotFound: (Get-CIMInstance:String) [], CommandNotFoundException
+ FullyQualifiedErrorId : CommandNotFoundException
#>
write-host "$serviceName seems to be owned by $serviceRunas. Now seizing permissions...";
# Grant permissions of service to the Administrators group
$nullOutput=PSExec -s -accepteula SetACL.exe -on $serviceName -ot srv -actn ace -ace 'n:Administrators;p:full' 2>&1; #redirect (>) 'stderr'(2) messages to 'stdout'(1); where 1 is a file descriptor (&), not a file
write-host "Process name $serviceName has been granted access to the Administrators group.";
# Retry stopping service
try{
Stop-Service $serviceName -Force;
write-host "$serviceName has been stopped successfully.";
return $true;
}
catch{
write-host $Error;
write-host "$serviceName has NOT been stopped successfully.";
return $false;
}
}
}else{
write-host "$serviceName is already stopped.";
return $true;
}
}
$matches=get-service|?{$_.DisplayName -like "*$serviceName*" -or $_.Name -like "*$serviceName*" -or $_.Servicename -like "*$serviceName*"};
#$matches=get-service|?{$_.DisplayName -like "*$serviceName*" -or $_.Name -like "*$serviceName*"};
if (!($matches)){
$message="$serviceName doesn't match anything on $env:computername";
write-host $message;
return $false;
}
if($matches.count -gt 1){
# This is a PowerShell 2.0 backward compatible technique to rebuild an object with a new column of Index values
$displayMatches=for ($i=0;$i -lt $matches.count;$i++){
$matches[$i]|Select-Object @{name='Index';e={$i}},Name,Displayname,Status
}
$displayMatches=$displayMatches|ft -AutoSize|Out-String
write-host "We have multiple matches for the $servicename`:`r`n-----------------------------------------------------------------$displayMatches";
$input=Read-Host "Please pick an index number from the above list"
write-host "Index value $input received.";
$matches=$matches[$input];
if (!($matches)){
write-host "Index value $input is invalid. No actions were taken.";
return $false;
}else{
$service=$matches;
forceKillService $service;
return $true;
}
}else{
$service=$matches;
forceKillService $service;
return $true;
}
}
function initPsSessionAsAdmin($computerName){
function checkAdminPrivileges{
param($credential)
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$currentUser=$myWindowsID.Name
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator;
if ($myWindowsPrincipal.IsInRole($adminRole))
{
write-host "$currentUser is an Administrator. Program will now initiate a new session with such account...";
$Host.UI.RawUI.BackgroundColor = "Black";
return $true;
}
else
{
return $false;
}
}
function getAdminCredentials{
$exitLoop=$false;
do {
$credential= get-credential;
if(checkAdminPrivileges -credential $credential){$exitLoop=$true};
sleep 1;
}while ($exitLoop -eq $false)
return $credential;
}
if (!(checkAdminPrivileges)){
$cred=getAdminCredentials
try{
$session=New-PSSession -Credential $cred -ComputerName $computerName;
}catch{
# Enable WinRM as try block command didn't succeed;
start-process powershell.exe -credential $cred -nonewwindow -ArgumentList "enable-psremoting -force";
refreshenv;
$session=New-PSSession -Credential $cred -ComputerName $computerName;
}
}else{
try{
$session=New-PSSession -ComputerName $computerName;
}catch{
# Enable WinRM as try block command didn't succeed;
start-process powershell.exe -credential $cred -nonewwindow -ArgumentList "winrm quickconfig -force;"
refreshenv;
$session=New-PSSession -ComputerName $computerName;
}
}
if ($session){
return $session;
}else{
write-host "Could not proceed due to errors in the process of initiating a new PSSession.";
break;
}
}
function invokeKillCommand{
param($service)
# Cleanup any lingering PS Sessions
get-pssession|remove-pssession;
if (!$adminPsSession -or $adminPsSession.State -eq "Closed"){$adminPsSession=initPsSessionAsAdmin -computerName $computerName;}
if(!(get-command psexec -ea SilentlyContinue)){
if (!(get-command choco -ea SilentlyContinue)) {
write-host "Installing Choco...";
Set-ExecutionPolicy Bypass -Scope Process -Force;
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'));
}
write-host "Installing PSExec...";
choco install sysinternals -y -force;
}
# Setting WinRM memory size to ensure success
#Correct format: winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="$ramMB"}'
#Alternaltive: psexec \\$computerName PowerShell Set-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB 1024 #This one runs into permission issues
[int]$ramMB=(gwmi -Class win32_operatingsystem -computername $computerName -ea stop).TotalVisibleMemorySize/1024
if ($ramMB -ge 4090){$ramMB=4090}
$nullOutput=invoke-expression "psexec \\$computerName -s winrm.cmd set winrm/config/winrs '@{MaxMemoryPerShellMB=`"$ramMB`"}'" 2>&1;
invoke-command -Session $adminPsSession -ScriptBlock{
param($importedFunc,$x)
[ScriptBlock]::Create($importedFunc).invoke($x);
} -args ${function:executeKillCommand},$service
Remove-PSSession $adminPsSession;
}
function killService{
param($service)
$computerNameRegex='^(.*?)\.'
$local=$([void]($computerName -match $computerNameRegex);if($matches){$matches[1]}else{$computerName}) -like $env:computername;
if ($local){
executeKillCommand -service $service;
}else{
invokeKillCommand -service $service;
}
}
killService -service $serviceName
}
function quickEmcopy{
param(
$sourceDirectory,
$destinationDirectory,
$logFile
)
$timer=[System.Diagnostics.Stopwatch]::StartNew();
$log="/LOG+:$logFile";
$message="=========================QuickEmcopy is starting on $env:computername at $(get-date -Format "yyyy-MM-dd-hh:mm:ss")=========================`r`n=========================Source: $sourceDirectory => Destination: $destinationDirectory=========================";
write-host $message;
Add-Content $logFile $message;
function installEmcopy{
$emcopyIsInstalled=(Get-Command emcopy.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
if (!($emcopyIsInstalled)){
$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$emCopySource = "https://blog.kimconnect.com/wp-content/uploads/2019/08/emcopy.zip";
$destinationFile = "$tempDir\emcopy.zip";
try{[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12}catch{}
New-Item -ItemType Directory -Force -Path $tempDir
New-Item -ItemType Directory -Force -Path $extractionDir
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($emCopySource,$destinationFile);
expandZipfile $destinationFile -Destination $extractionDir
}else{
"EMCOPY is currently available in this system.`n";
}
}
if(!(Get-Command emcopy.exe -ea silentlycontinue)){installEmcopy};
$sourceParentFolder=split-path $sourceDirectory -Parent
$expression="emcopy.exe '$sourceDirectory' '$destinationDirectory' $emcopySwitches /xd '$sourceParentFolder`System Volume Information' '$sourceParentFolder`$RECYCLE.BIN'";
try{
write-host $expression;
Invoke-Expression $expression;
}
catch{
write-host "There were errors`r`n $Error";
}
$hoursElapsed=[math]::round($timer.Elapsed.TotalHours,2);
$message="`r`n=========================QuickEmcopy has completed at $(get-date -Format "yyyy-MM-dd-hh:mm:ss") - Total Hours: $hoursElapsed=========================";
Add-Content $logFile $message;
return $hoursElapsed;
}
function proceed{
$osAndMemory = gwmi -Class win32_operatingsystem -ea stop| select @{Name="os";Expression={$_.Caption}},`
@{Name="Memory";Expression={$_.TotalVisibleMemorySize / 1048576}},`
@{Name="FreeMemory";Expression={$_.FreePhysicalMemory / 1048576}},`
@{Name = "Utilization"; Expression = {"{0:N2} %" -f ((($_.TotalVisibleMemorySize - $_.FreePhysicalMemory)*100)/ $_.TotalVisibleMemorySize) }}
if($osAndMemory){
#$os=$osAndMemory.os;
$memory=$osAndMemory.Memory;
$freeMemory=$osAndMemory.FreeMemory;
$memoryUtilization=$osAndMemory.Utilization;
}
$osName=(Get-WmiObject -class Win32_OperatingSystem).Caption;
$windowsVersionNumber=[System.Environment]::OSVersion.Version.Major;
$arrLength=$arr.Length
$robocopyVersion=getExecutableVersion
$emcopyVersion=getExecutableVersion -executablename "emcopy.exe";
$initInfo="=============================================Job Started: $dateStamp=============================================`r`n";
$initInfo+="Source server name: $ENV:COMPUTERNAME`n";
$initInfo+="$osName`r`nPowershell version: $powerShellVersion`r`n$robocopyVersion`r`n$emcopyVersion`r`n"
$initInfo+="Memory: $memory GB | Free: $freeMemory GB | Utilization: $memoryUtilization`r`n`r`n"
$initInfo+="Now processing the following operations:`r`n";
$initInfo+=for ($i=0;$i -lt $arrLength; $i++){
$from=$arr[$i].From;
$to=$arr[$i].To;
$passNumber=$i+1;
"$passNumber`: $from => $to`r`n";
}
Write-Host $initInfo;
if(!(test-path '$logPath' -ea SilentlyContinue)){New-Item -ItemType Directory -Force -Path $logPath | Out-Null}
Add-Content $logFile $initInfo;
function finalSync{
#Final Sync:
#1. Disable File Shares on the primary NIC, and remove all SMB sessions
#2. Perform Quick Sync using Emcopy. (Perform File Checks using WinMerge after cutover)
#4. Send an email to notify Admins of final-sync completion
$finalSyncTimer=[System.Diagnostics.Stopwatch]::StartNew();
stopAnyService -serviceName "MsMpSvc";
$initMessage="=======================================Final-sync has initiated for $env:computername at $(get-date)=======================================`r`n";
Add-Content $logFile $initMessage;
sendEmailViaRelay -emailMessage $initMessage;
# Check firewall Statuses
$firewallAllProfiles=netsh advfirewall show allprofiles |?{$_ -match "State"}
$firewallDomainState=$([void]($firewallAllProfiles[0] -match "(ON|OFF)");$matches[1];)
$firewallPrivateState=$([void]($firewallAllProfiles[1] -match "(ON|OFF)");$matches[1];)
$firewallPublicState=$([void]($firewallAllProfiles[2] -match "(ON|OFF)");$matches[1];)
# Disable File and Printer Sharing
<# These methods don't work because some servers are controlled by Group Policy
$hive="REGISTRY::HKLM\SOFTWARE\Policies\Microsoft\WindowsFirewall\DomainProfile\Services\FileAndPrint";
$key="Enabled";
$value=0
# Disable
Set-Itemproperty -path $hive -Name $key -value $value
# Enable
Set-Itemproperty -path $hive -Name $key -value 1
# Set-NetFirewallRule -DisplayGroup "File And Printer Sharing" -Enabled False
# Doesn't work with PowerShell 2.0# netsh advfirewall firewall set rule group="File and Printer Sharing" new enable=No
# netsh firewall set service type=fileandprint mode=enable profile=all
#>
netsh advfirewall set allprofiles state on
netsh advfirewall firewall add rule name='BlockSMB' protocol=TCP dir=in localport=445 action=block
# Disconnect all connected SMB Sessions
do {
$disconnectResult=net session /delete /y 2>&1
$nullOutput=ping -n 1 127.0.0.1 2>&1;
} while($disconnectResult.count -ne 2)
for ($i=0;$i -lt $arrLength; $i++){
$from=$arr[$i].From;
$to=$arr[$i].To;
$passNumber=$i+1;
$processDisplay="=======================================Pass $passNumber of $arrLength`: $from => $to=======================================`r`n";
Add-Content $logFile $processDisplay;
$thisDuration=quickEmcopy -sourceDirectory $from -destinationDirectory $to -logFile $logFile
$iterationCompleteMessage="=======================================Pass $passNumber of $arrLength` COMPLETED in $thisDuration hours $from => $to=======================================`r`n";
Add-Content $logFile $iterationCompleteMessage;
}
# Revert firewall back to previous settings
# Set-NetFirewallRule -DisplayGroup "File And Printer Sharing" -Enabled True # PowerShell 4.0 required
# netsh advfirewall firewall set rule group="File and Printer Sharing" new enable=Yes
netsh advfirewall firewall delete rule name='BlockSMB';
netsh advfirewall set domainprofile state $firewallDomainState;
netsh advfirewall set privateprofile state $firewallPrivateState;
netsh advfirewall set publicprofile state $firewallPublicState;
#netsh advfirewall set allprofiles state off
$closingMessage="Final-sync has completed for $env:computername at $(get-date). Total run time: $([math]::round($finalSyncTimer.Elapsed.TotalHours,2)) hours.`r`n"
$closingMessage+="Please check the log at this location: $logFile`r`n";
write-host $closingMessage;
Add-Content $logFile $closingMessage;
sendEmailViaRelay -emailMessage $closingMessage -logAttachment $logFile;
}
function syncWithReport{
# Process the copying operations for Windows 2008+ where robocopy version XP027 or higher is available
$previousDriveLetter="none"
$thisDriveLetter="none"
$nextDriveLetter="none"
for ($i=0;$i -lt $arrLength; $i++){
$from=$arr[$i].From;
$nextFrom=$arr[$i+1].From ;
$to=$arr[$i].To;
$passNumber=$i+1;
$processDisplay="=======================================Pass $passNumber of $arrLength`: $from => $to=======================================`r`n";
Add-Content $logFile $processDisplay;
$thisDriveLetter=Split-Path -Path $from -Qualifier
$nextDriveLetter=try{Split-Path -Path $nextFrom -Qualifier}catch{$false;}
"Previous drive $previousDriveLetter vs current $thisDriveLetter vs next $nextDriveLetter"
if($previousDriveLetter -ne $thisDriveLetter){
$toCreateSnapshot=$True;
}else{
$toCreateSnapshot=$False;
}
if($nextDriveLetter -ne $thisDriveLetter){
$toDeleteSnapshot=$True;
}else{
$toDeleteSnapshot=$False;
}
write-host "startCopy $from $to";
copyWithoutReport -source $from -destination $to -takeSnapshot $toCreateSnapshot -removeSnapshot $toDeleteSnapshot;
$previousDriveLetter=$thisDriveLetter;
$iterationCompleteMessage="=======================================Pass $passNumber of $arrLength` COMPLETED $from => $to=======================================`r`n";
Add-Content $logFile $iterationCompleteMessage;
}
# Record overall speed to log
$aggregateHours=[math]::Round($totalTime/3600,2);
$aggregateSpeed=[math]::Round(($totalStorage/$totalTime)*3600,2);
$totalStorage=[math]::Round($totalStorage,2);
$summary="`r`n==============================Storage: $totalStorage GiB | Time: $aggregateHours hours | Speed: $aggregateSpeed GiB/hr ==============================`r"
write-host $summary;
Add-Content $logFile $summary;
generateErrorsLog;
}
function syncWithoutReport{
# Process the copying operations for Windows 2008+ where robocopy version XP027 or higher is available
$previousDriveLetter="none"
$thisDriveLetter="none"
$nextDriveLetter="none"
for ($i=0;$i -lt $arrLength; $i++){
$from=$arr[$i].From;
$nextFrom=$arr[$i+1].From ;
$to=$arr[$i].To;
$passNumber=$i+1;
$processDisplay="=======================================Pass $passNumber of $arrLength`: $from => $to=======================================`r`n";
Add-Content $logFile $processDisplay;
$thisDriveLetter=Split-Path -Path $from -Qualifier
$nextDriveLetter=try{Split-Path -Path $nextFrom -Qualifier}catch{$false;}
"Previous drive $previousDriveLetter vs current $thisDriveLetter vs next $nextDriveLetter"
if($previousDriveLetter -ne $thisDriveLetter){
$toCreateSnapshot=$True;
}else{
$toCreateSnapshot=$False;
}
if($nextDriveLetter -ne $thisDriveLetter){
$toDeleteSnapshot=$True;
}else{
$toDeleteSnapshot=$False;
}
write-host "startCopy $from $to";
startCopy -source $from -destination $to -takeSnapshot $toCreateSnapshot -removeSnapshot $toDeleteSnapshot;
$previousDriveLetter=$thisDriveLetter;
$iterationCompleteMessage="=======================================Pass $passNumber of $arrLength` COMPLETED $from => $to=======================================`r`n";
Add-Content $logFile $iterationCompleteMessage;
}
# Record overall speed to log
$aggregateHours=[math]::Round($totalTime/3600,2);
$summary="`r`n==============================Time: $aggregateHours hours==============================`r"
write-host $summary;
Add-Content $logFile $summary;
generateErrorsLog;
}
function simpleRoboCopy{
# Process the copying operations for Windows 2003, 2000, & NT4
$timer2=[System.Diagnostics.Stopwatch]::StartNew();
for ($i=0;$i -lt $arrLength; $i++){
$from=$arr[$i].From;
$to=$arr[$i].To;
$passNumber=$i+1;
$processDisplay="=======================================Pass $passNumber of $arrLength`: $from => $to=======================================`r`n";
Add-Content $logFile $processDisplay;
#$commandString="robocopy '$from' '$to' /TBD /FFT /NS /NC /NDL /S /E /COPY:DATS /R:0 /W:0 $log";
$commandString="robocopy '$from' '$to' /TBD /FFT /NS /NC /NDL /S /E /COPY:DATS /PURGE /MIR /B /NP /XO /R:0 /W:0 $log"
write-host $commandString;
#cmd /c pause | out-null;
Invoke-Expression $commandString
$iterationCompleteMessage="=======================================Pass $passNumber of $arrLength` COMPLETED $from => $to=======================================`r`n";
Add-Content $logFile $iterationCompleteMessage;
}
$totalHours=[math]::Round($timer2.Elapsed.TotalHours,2);
$summary="`r`n==============================Total Hours: $totalHours==============================`r"
Add-Content $logFile $summary;
}
if ($finalSync){
finalSync;
}else{
if ($windowsVersionNumber -ge 6){
$initialStorageTimer=[System.Diagnostics.Stopwatch]::StartNew();
$initialTotalSize=0;
$initFolderSizeError=$false;
$message="Estimating total storage to be copied...`r`n";
write-host $message;
Add-Content $logFile $message;
for ($i=0;$i -lt $arrLength; $i++){
$thisFolder=$arr[$i].From;
$message="Obtaining folder size for $thisFolder...`r`n";
write-host $message;
Add-Content $logFile $message;
$thisFolderSize = (Get-FolderSize $thisFolder).TotalGBytes;
if ($thisFolderSize -ne $null){
$initialTotalSize+=$thisFolderSize;
}else{
$message="Unable to get folder size of $thisFolder.";
write-host $message;
Add-Content $logFile $message;
$initFolderSizeError=$true;
break;
}
}
$initialStorageCalculationTime=[math]::Round($initialStorageTimer.Elapsed.TotalHours,2);
$initialStorageTimer.Stop();
$initialStorage=[math]::Round($initialTotalSize,2);
$message="Initial Storage Calculation is $initialStorage GB. ($initialStorageCalculationTime hours)"
write-host $message;
Add-Content $logFile $message;
if ($freeMemory -ge 4 -and $initialTotalSize -le 5120 -and !$initFolderSizeError) {
$message="Syncing with additional stats collection reporting.";
write-host $message;
Add-Content $logFile $message
syncWithReport;
}else{
$message="Syncing without collecting additional stats.";
write-host $message;
Add-Content $logFile $message
syncWithoutReport;
}
}else{
simpleRoboCopy;
}
}
}
if($arr -ne $null){proceed;}else{write-host "Program will not proceed without valid sources and destinations."}
################################## Main Programming Sequence ####################################
#pause;
#cmd /c pause | out-null;
Categories: