PowerShell: Perform Final Sync Between 2 Directories

Current Version

# Final-Sync.ps1
# This function performs CRC checks on each file at the source.
# For every mismatch of file hashing signatures at the destination, the source file will be copied over to the destination.
# A log file will also be set to record progress

$sourceDirectory="E:\ShomeFolder"
$destinationDirectory="\\FiletSherver999\ShomeFolder"
$logFile="c:\final-sync-log.txt"
$log="/LOG+:$logFile"
$quickEmcopySwitches="/noprivchecks /de /s /purge /r:0 /w:0 $log";

# This system backup privilege escalation should be ran only once per session
function addSystemPrivilege{
    param(
        [String[]]$privileges=@("SeBackupPrivilege","SeRestorePrivilege")
    )

    function includeSystemPrivileges{
    $win32api = @'
    using System;
    using System.Runtime.InteropServices;

    namespace SystemPrivilege.Win32API
    {
      [StructLayout(LayoutKind.Sequential)]
      public struct LUID
      {
        public UInt32 LowPart;
        public Int32 HighPart;
      }

      [StructLayout(LayoutKind.Sequential)]
      public struct LUID_AND_ATTRIBUTES
      {
        public LUID Luid;
        public UInt32 Attributes;
      }

      [StructLayout(LayoutKind.Sequential)]
      public struct TOKEN_PRIVILEGES
      {
        public UInt32 PrivilegeCount;
        public LUID Luid;
        public UInt32 Attributes;
      }

      public class Privileges
      {
        public const UInt32 DELETE = 0x00010000;
        public const UInt32 READ_CONTROL = 0x00020000;
        public const UInt32 WRITE_DAC = 0x00040000;
        public const UInt32 WRITE_OWNER = 0x00080000;
        public const UInt32 SYNCHRONIZE = 0x00100000;
        public const UInt32 STANDARD_RIGHTS_ALL = (
                                                    READ_CONTROL |
                                                    WRITE_OWNER |
                                                    WRITE_DAC |
                                                    DELETE |
                                                    SYNCHRONIZE
                                                );
        public const UInt32 STANDARD_RIGHTS_REQUIRED = 0x000F0000u;
        public const UInt32 STANDARD_RIGHTS_READ = 0x00020000u;

        public const UInt32 SE_PRIVILEGE_ENABLED_BY_DEFAULT = 0x00000001u;
        public const UInt32 SE_PRIVILEGE_ENABLED = 0x00000002u;
        public const UInt32 SE_PRIVILEGE_REMOVED = 0x00000004u;
        public const UInt32 SE_PRIVILEGE_USED_FOR_ACCESS = 0x80000000u;

        public const UInt32 TOKEN_QUERY = 0x00000008;
        public const UInt32 TOKEN_ADJUST_PRIVILEGES = 0x00000020;

        public const UInt32 TOKEN_ASSIGN_PRIMARY = 0x00000001u;
        public const UInt32 TOKEN_DUPLICATE = 0x00000002u;
        public const UInt32 TOKEN_IMPERSONATE = 0x00000004u;
        public const UInt32 TOKEN_QUERY_SOURCE = 0x00000010u;
        public const UInt32 TOKEN_ADJUST_GROUPS = 0x00000040u;
        public const UInt32 TOKEN_ADJUST_DEFAULT = 0x00000080u;
        public const UInt32 TOKEN_ADJUST_SESSIONID = 0x00000100u;
        public const UInt32 TOKEN_READ = (
                                          STANDARD_RIGHTS_READ |
                                          TOKEN_QUERY
                                       );
        public const UInt32 TOKEN_ALL_ACCESS = (
                                                STANDARD_RIGHTS_REQUIRED |
                                                TOKEN_ASSIGN_PRIMARY |
                                                TOKEN_DUPLICATE |
                                                TOKEN_IMPERSONATE |
                                                TOKEN_QUERY |
                                                TOKEN_QUERY_SOURCE |
                                                TOKEN_ADJUST_PRIVILEGES |
                                                TOKEN_ADJUST_GROUPS |
                                                TOKEN_ADJUST_DEFAULT |
                                                TOKEN_ADJUST_SESSIONID
                                             );

        [DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
        public static extern IntPtr GetCurrentProcess();

        [DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
        public static extern IntPtr GetCurrentThread();

        [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
        public static extern bool LookupPrivilegeValue(string lpSystemName, string lpName, out LUID lpLuid);

        [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
        public static extern bool AdjustTokenPrivileges(IntPtr TokenHandle, bool DisableAllPrivileges, ref TOKEN_PRIVILEGES NewState, UInt32 BufferLengthInBytes, IntPtr PreviousStateNull, IntPtr ReturnLengthInBytesNull);

        [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
        public static extern bool OpenProcessToken(IntPtr ProcessHandle, UInt32 DesiredAccess, out IntPtr TokenHandle);

        [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
        public static extern bool OpenThreadToken(IntPtr ThreadHandle, UInt32 DesiredAccess, bool OpenAsSelf, out IntPtr TokenHandle);

        [DllImport("ntdll.dll", EntryPoint = "RtlAdjustPrivilege")]
        public static extern int RtlAdjustPrivilege(
                    UInt32 Privilege,
                    bool Enable,
                    bool CurrentThread,
                    ref bool Enabled
                    );

        [DllImport("Kernel32.dll", SetLastError = true)]
        public static extern bool CloseHandle(IntPtr handle);

        //
        //

        private static LUID LookupPrivilege(string privilegeName)
        {
          LUID privilegeValue = new LUID();
      
          bool res = LookupPrivilegeValue(null, privilegeName, out privilegeValue);

          if (!res)
          {
            throw new Exception("Error: LookupPrivilegeValue()");
          }

          return privilegeValue;
        }

        //
        //

        public static void AdjustPrivilege(string privilegeName, bool enable)
        {
          IntPtr accessToken = IntPtr.Zero;
          bool res = false;

          try
          {
            LUID privilegeValue = LookupPrivilege(privilegeName);

            res = OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, false, out accessToken);
        
            if (!res)
            {
              res = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, out accessToken);

              if (!res)
              {
                throw new Exception("Error: OpenProcessToken()");
              }
            }

            TOKEN_PRIVILEGES tokenPrivileges = new TOKEN_PRIVILEGES();
            tokenPrivileges.PrivilegeCount = 1;
            tokenPrivileges.Luid = privilegeValue;

            if (enable)
            {
              tokenPrivileges.Attributes = SE_PRIVILEGE_ENABLED;
            }
            else
            {
              tokenPrivileges.Attributes = 0;
            }

            res = AdjustTokenPrivileges(accessToken, false, ref tokenPrivileges, (uint)System.Runtime.InteropServices.Marshal.SizeOf(tokenPrivileges), IntPtr.Zero, IntPtr.Zero);
        
            if (!res)
            {
              throw new Exception("Error: AdjustTokenPrivileges()");
            }
          }

          finally
          {
            if (accessToken != IntPtr.Zero)
            {
              CloseHandle(accessToken);
              accessToken = IntPtr.Zero;
            }
          }
        }
      }
    }
'@

  if ([object]::Equals(('SystemPrivilege.Win32API.Privileges' -as [type]), $null)) {
    Add-Type -TypeDefinition $win32api
    }
}
    includeSystemPrivileges;


    $privileges|%{[SystemPrivilege.Win32API.Privileges]::AdjustPrivilege($_, $true)}

    # Validation
    whoami /priv|?{$_ -match "SeBackupPrivilege|SeRestorePrivilege"}
}
addSystemPrivilege;

function finalSync{
    param(
        $sourceDirectory,
        $destinationDirectory,
        $protocol="crc",
        $logFile
        )
    $message="=========================Starting to compare using protocol $protocol on $env:computername at $(get-date -Format "yyyy-MM-dd-hh:mm:ss")=========================`r`n";
    Add-Content $logFile $message;

    if ($protocol -like "crc"){
        function expandZipfile($file, $destination){
            $shell = new-object -com shell.application
            $zip = $shell.NameSpace($file)
    
            foreach($item in $zip.items()){
            $shell.Namespace($destination).copyhere($item)
            }
        }

        $crcInstalled=(Get-Command getcrc.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
        if (!($crcInstalled)){
            $tempDir="C:\Temp";
            $extractionDir="C:\Windows"
            $sourceFile = "https://blog.kimconnect.com/wp-content/uploads/2020/01/getcrc.zip";
            $destinationFile = "$tempDir\getcrc.zip";
            try{[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12}catch{}
            New-Item -ItemType Directory -Force -Path $tempDir
            New-Item -ItemType Directory -Force -Path $extractionDir
            $webclient = New-Object System.Net.WebClient;
            $WebClient.DownloadFile($sourceFile,$destinationFile);
            expandZipfile $destinationFile -Destination $extractionDir
        }
    }

    $accumulatedTime=0;
    $timer=[System.Diagnostics.Stopwatch]::StartNew();
    $sourceItems=Get-ChildItem $sourceDirectory -Recurse -Force -ea SilentlyContinue; # Include hidden files
    $sourceFiles=$sourceItems|?{!$_.PSisContainer}; # Exclude folders
    $sourceFilePaths=$sourceFiles|%{$_.FullName};
    $filesCount=$sourceFilePaths.Count;
    $rootCount=$sourceDirectory.Length;
    $commonDenominatingPaths=$sourceFilePaths|%{$_.substring($rootCount,$_.length-$rootCount)};
    $destinationFilePaths=$commonDenominatingPaths | %{if([string]$_[0] -eq "\"){
                                                                                [string]$destinationDirectory+[string]$_;
                                                                                    }else{
                                                                                        [string]$destinationDirectory+"\"+[string]$_;
                                                                                        }
                                                                                }    

    $filesCountTime+=$timer.Elapsed.TotalMinutes;
    $message="$filesCount files detected at $sourceDirectory..."
    write-host $message;
    Add-Content $logFile $message;

    function getHash{
        param($file,$protocol="crc")   

        switch ($protocol){
            "crc"{
                return \\snapshots\FileServerClusters\Software\crc32.exe '$file'
                }
            "md5"{return $($md5 = (New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider)
                                try{
                                $stream = [System.IO.File]::Open("$file",[System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
                                $hash = [System.BitConverter]::ToString($md5.ComputeHash($stream));
                                $stream.Dispose();
                                $stream.Close();      
                                return $hash;
                                }
                                catch{
                                write-host "Cannot get MD5 hash of $file";
                                return $hash});                    
                    }
            }
        }

    $timer.Reset(); # Windows 2008 timer doesn't have the restart() method
    $timer.Start();
    for ($i=0;$i -lt $filesCount;$i++){
        # Compare each source file with its counterpart incrementally
        $sourceFile=$sourceFilePaths[$i];
        $destinationFile=$destinationFilePaths[$i];
        $sourceHash=getHash -file $sourceFile -protocol $protocol;
        $destinationHash=getHash -file $destinationFile -protocol $protocol;
        $match=$sourceHash -eq $destinationHash
        if(!$match){
            $message="$sourceFile has been copied to $destinationFile due to a mismatch.`r`n";
            write-host $message;
            $sourceFolder=split-path $sourceFile -Parent
            $destinationFolder=split-path $destinationFile -Parent
            $fileName=split-path $sourceFile -Leaf          
            $nullOutput = robocopy $sourceFolder $destinationFolder $fileName /is 2>&1;
            Add-Content $logFile $message;
            }
        # exclude this feature to improve speed slightly
        # Output a periodic progress report every 5 minutes
        if($timer.Elapsed.TotalMinutes -ge 5){
            $accumulatedTime+=$timer.Elapsed.TotalMinutes;
            $percent=($i/$filesCount).tostring("P")
            $message="$([math]::round($accumulatedTime,2)) minutes: $($i+1) of $filesCount processed `($percent`).";
            Add-Content $logFile $message;
            $timer.Reset(); # Windows 2008 timer doesn't have the restart() method
            $timer.Start();
            }
        #
        }
    $totalTime+=$accumulatedTime+$filesCountTime;
    $message="$sourceDirectory vs $destinationDirectory`r`n=========================File compare is completed on $env:computername. TotalElapsed: $([math]::round($totalTime,2)) minutes=========================";
    Add-Content $logFile $message;
}

finalSync -sourceDirectory $source -destinationDirectory $destination -protocol "crc" -logFile $logFile

# Final-Sync.ps1
# This function performs CRC checks on each file at the source.
# For every mismatch of file hashing signatures at the destination, the source file will be copied over to the destination.
# A log file will also be set to record progress
# Word of advise: don't run this from a slow computer against 100 million files.

function finalSync{
    param(
        $sourceDirectory,
        $destinationDirectory,
        $logFile="C:\final-sync-log.txt"
        )

    $sourceItems=Get-ChildItem $sourceDirectory -Recurse -Force; # Include hidden files
    $sourceFiles=$sourceItems|?{!$_.PSisContainer}; # Exclude folders
    $sourceFilePaths=$sourceFiles|%{$_.FullName};
    $filesCount=$sourceFilePaths.Count;
    $rootCount=$sourceDirectory.Length;
    $commonDenominatingPaths=$sourceFilePaths|%{$_.substring($rootCount,$_.length-$rootCount)};
    $destinationFilePaths=$commonDenominatingPaths | %{if([string]$_[0] -eq "\"){
                                                                                [string]$destinationDirectory+[string]$_;
                                                                                    }else{
                                                                                        [string]$destinationDirectory+"\"+[string]$_;
                                                                                        }
                                                                                }
    # Get-FileHash (PowerShell 4.0+) replacement for Windows 2008. Forward-compatible.
    function getMd5{
        param(
            $file,
            $md5 = (New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider)
            )
    
        # $hash = [System.BitConverter]::ToString($md5.ComputeHash([System.IO.File]::ReadAllBytes($file))); # This line has the issue of 2GB file size limit    
        try{
        $stream = [System.IO.File]::Open("$file",[System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
        $hash = [System.BitConverter]::ToString($md5.ComputeHash($stream));
        $stream.Dispose();
        $stream.Close();      
        return $hash;
        }
        catch{
        return $null;
        }
    }
    
    $message="Starting the file compare progress...";
    Add-Content $logFile $message;
    $timer=[System.Diagnostics.Stopwatch]::StartNew();
    $md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider;
    for ($i=0;$i -lt $filesCount;$i++){
        # Compare each source file with its counterpart incrementally
        $sourceFile=$sourceFilePaths[$i];
        $destinationFile=$destinationFilePaths[$i];
        $sourceHash=getMd5 -file $sourceFile -md5 $md5;
        $destinationHash=getMd5 -file $destinationFile -md5 $md5;
        $match=$sourceHash -eq $destinationHash
        if(!$match){
            $message="$sourceFile has been copied to $destinationFile`r`n";
            write-host $message;
            $sourceFolder=split-path $sourceFile -Parent
            $destinationFolder=split-path $destinationFile -Parent
            $fileName=split-path $sourceFile -Leaf          
            $nullOutput = robocopy $sourceFolder $destinationFolder $fileName /is 2>&1;
            Add-Content $logFile $message;
            }
        
        # Output a periodic progress report every 10 minutes
        if($timer.Elapsed.TotalMinutes -ge 10){
            $accumulatedTime+=$timer.Elapsed.TotalMinutes;
            $percent=($i/$filesCount).tostring("P")
            $message="$($i+1) of $filesCount processed in $([math]::round($accumulatedTime,2)) `($percent`) minutes.";
            Add-Content $logFile $message;
            $timer.Restart();
            }
        }
}

Output:

<# Sample Usage:
PS C:\Windows\system32> finalSync -sourceDirectory c:\temp -destinationDirectory c:\tempcopy -logFile C:\log.txt
C:\temp\helloworld.py has been copied to c:\tempcopy\helloworld.py
#>