Private/Core.ZipWriter.ps1
|
# Core helper: fast zip creation for many small files. # # Opening a file is expensive on machines with real-time antivirus (every open is scanned), and a zip # archive can only be written by one thread. EdgeZipJob therefore reads the source files on several # threads (into memory, bounded) while a single writer compresses them into the archive. The job runs in # the background; PowerShell polls its counters to drive Write-Progress. $script:EdgeZipWriterSource = @' using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Threading; using System.Threading.Tasks; namespace EdgeProfileSync { public sealed class ZipSkippedFile { public string Source { get; set; } public string Entry { get; set; } public string Reason { get; set; } } public sealed class EdgeZipJob { // Files up to this size are read on the reader threads; larger ones are streamed by the writer. private const long PrefetchLimit = 16L * 1024 * 1024; private long _bytesDone; private int _filesDone; private volatile string _current = string.Empty; private readonly ConcurrentQueue<ZipSkippedFile> _skipped = new ConcurrentQueue<ZipSkippedFile>(); public long TotalBytes { get; private set; } public int TotalFiles { get; private set; } public long BytesDone { get { return Interlocked.Read(ref _bytesDone); } } public int FilesDone { get { return Volatile.Read(ref _filesDone); } } public string Current { get { return _current; } } public Task Task { get; private set; } public ZipSkippedFile[] Skipped { get { return _skipped.ToArray(); } } private sealed class Item { public int Index; public byte[] Data; // null when streamed by the writer public string Error; } public static EdgeZipJob Start(string zipPath, string[] sources, string[] entries, long[] sizes, int readers) { if (sources.Length != entries.Length || sources.Length != sizes.Length) throw new ArgumentException("sources, entries and sizes must have the same length"); var job = new EdgeZipJob(); job.TotalFiles = sources.Length; long total = 0; foreach (var s in sizes) total += s; job.TotalBytes = total; job.Task = Task.Run(() => job.Run(zipPath, sources, entries, sizes, Math.Max(1, readers))); return job; } private static FileStream OpenShared(string path) { return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 81920, FileOptions.SequentialScan); } private void Run(string zipPath, string[] sources, string[] entries, long[] sizes, int readers) { using (var queue = new BlockingCollection<Item>(boundedCapacity: 256)) { var producer = Task.Run(() => { try { Parallel.For(0, sources.Length, new ParallelOptions { MaxDegreeOfParallelism = readers }, i => { var item = new Item { Index = i }; if (sizes[i] <= PrefetchLimit) { try { using (var fs = OpenShared(sources[i])) using (var ms = new MemoryStream((int)Math.Max(0, Math.Min(sizes[i], int.MaxValue)))) { fs.CopyTo(ms); item.Data = ms.ToArray(); } } catch (Exception ex) { item.Error = ex.Message; } } queue.Add(item); }); } finally { queue.CompleteAdding(); } }); using (var stream = new FileStream(zipPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, false)) { foreach (var item in queue.GetConsumingEnumerable()) { int i = item.Index; _current = entries[i]; if (item.Error != null) { _skipped.Enqueue(new ZipSkippedFile { Source = sources[i], Entry = entries[i], Reason = item.Error }); } else { try { var entry = zip.CreateEntry(entries[i], CompressionLevel.Fastest); try { entry.LastWriteTime = File.GetLastWriteTime(sources[i]); } catch { } using (var output = entry.Open()) { if (item.Data != null) output.Write(item.Data, 0, item.Data.Length); else using (var fs = OpenShared(sources[i])) fs.CopyTo(output); } } catch (IOException ex) when (item.Data == null) { // A large file vanished or got locked; the empty entry stays, report it. _skipped.Enqueue(new ZipSkippedFile { Source = sources[i], Entry = entries[i], Reason = ex.Message }); } } Interlocked.Add(ref _bytesDone, sizes[i]); Interlocked.Increment(ref _filesDone); } } producer.GetAwaiter().GetResult(); } } } } '@ function Initialize-EdgeZipWriter { # Compiles the zip writer once per session (about a second, on first use only). [CmdletBinding()] param() if ('EdgeProfileSync.EdgeZipJob' -as [type]) { return } Add-Type -TypeDefinition $script:EdgeZipWriterSource -Language CSharp -ErrorAction Stop } function Write-EdgeZipFromList { <# .SYNOPSIS Writes the given files into a new zip archive with parallel readers and a byte-based progress bar. .DESCRIPTION -Items: objects with Source (full path), Entry (path inside the zip, '/' separated) and Bytes. Files that cannot be read (vanished, locked) are skipped and returned as objects (Source, Entry, Reason); everything else ends up in the archive. #> [CmdletBinding()] param( [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Items, [Parameter(Mandatory)][string]$ZipPath, [string]$Activity = 'Creating zip archive', [int]$ProgressId = 2, [int]$Readers = 8 ) Initialize-EdgeZipWriter $sources = [string[]]@($Items | ForEach-Object { $_.Source }) $entries = [string[]]@($Items | ForEach-Object { $_.Entry }) $sizes = [long[]]@($Items | ForEach-Object { [long]$_.Bytes }) $job = [EdgeProfileSync.EdgeZipJob]::Start($ZipPath, $sources, $entries, $sizes, $Readers) $progress = New-EdgeProgress -Activity $Activity -Total $job.TotalBytes -Id $ProgressId try { while (-not $job.Task.Wait(250)) { Update-EdgeProgress -Progress $progress -Done $job.BytesDone ` -Status ('File {0:N0} of {1:N0}' -f $job.FilesDone, $job.TotalFiles) -CurrentOperation $job.Current } # Surfaces an exception of the background job (disk full, destination not writable, ...). $job.Task.GetAwaiter().GetResult() } finally { Complete-EdgeProgress -Progress $progress } $job.Skipped } |