Private/Import.ps1
|
Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' # zoxide import, mirroring zoxide 0.10.0 src/cmd/import.rs, src/import.rs (Importer, run) and the # importers in src/import/: z.rs (whose line iterator fasd.rs, zsh_z.rs and z_lua.rs reuse), # autojump.rs and atuin.rs. 0.10.0 takes no path argument: each importer reads its tool's own data # file (from that tool's environment variable or default location) or, for atuin, runs `atuin`. # # Record [pscustomobject] one parsed input record, either # Path, Rank, LastAccessed an entry for Database::add_unchecked, or # Error "<location>: <reason>", printed to stderr and skipped function Format-ZoDebugString { <# .SYNOPSIS A string in Rust's Debug form: double-quoted, with escapes. .DESCRIPTION Mirrors `{:?}` for str and Path (the `could not read {path:?}` and `{timestamp:?}` errors): backslash and double quote are escaped, tab, CR, LF and NUL use their short escapes, other control characters become \u{hex}. A single quote is not escaped. #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory)] [AllowEmptyString()] [string]$Text ) $builder = [Text.StringBuilder]::new('"') foreach ($ch in $Text.ToCharArray()) { # Switch on the code point, not the character: PowerShell's switch compares strings with the # culture, and under ICU control characters are ignorable, so ESC would match the NUL case. $escaped = switch ([int]$ch) { 92 { '\\' } 34 { '\"' } 9 { '\t' } 13 { '\r' } 10 { '\n' } 0 { '\0' } default { if ([char]::IsControl($ch)) { '\u{' + ([int]$ch).ToString('x') + '}' } else { [string]$ch } } } [void]$builder.Append($escaped) } $builder.Append('"').ToString() } function Get-ZoIoErrorText { <# .SYNOPSIS The text Rust's io::Error prints for a failed file open, from a .NET exception. .DESCRIPTION On Windows the Win32 code in the HRESULT gives the system message and '(os error N)', as Rust prints it ('The system cannot find the file specified. (os error 2)'). Elsewhere the common errno cases are named the way Rust names them; anything else keeps the .NET message. #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory)] [Exception]$Exception, [bool]$Windows = $IsWindows ) if ($Windows -and ($Exception.HResult -band 0xFFFF0000) -eq 0x80070000) { $code = $Exception.HResult -band 0xFFFF return '{0} (os error {1})' -f ([ComponentModel.Win32Exception]::new($code).Message), $code } if ($Exception -is [IO.FileNotFoundException] -or $Exception -is [IO.DirectoryNotFoundException]) { return 'No such file or directory (os error 2)' } if ($Exception -is [UnauthorizedAccessException]) { return 'Permission denied (os error 13)' } $Exception.Message } function Get-ZoImportDataPath { <# .SYNOPSIS The data file(s) an importer reads, in the order it tries them. .DESCRIPTION Mirrors data_path in each importer: z reads $_Z_DATA or ~/.z; fasd $_FASD_DATA or ~/.fasd; zsh-z $ZSHZ_DATA, then $_Z_DATA, then ~/.z; z.lua a non-empty $_ZL_DATA (off Windows, one that does not look like 'C:\' or 'C:/') or ~/.zlua, and then the fish location $XDG_DATA_HOME/zlua/zlua.txt or ~/.local/share/zlua/zlua.txt; autojump ~/Library/autojump/autojump.txt on macOS, %APPDATA%\autojump\autojump.txt on Windows (throws '%APPDATA% is not set' without it) and $XDG_DATA_HOME/autojump/autojump.txt or ~/.local/share/autojump/autojump.txt elsewhere. A variable that is set counts even when empty, except _ZL_DATA. The home directory is the user profile folder. #> [CmdletBinding()] [OutputType([string[]])] param( [Parameter(Mandatory)] [ValidateSet('autojump', 'fasd', 'z', 'z.lua', 'zsh-z')] [string]$Source, [bool]$Windows = $IsWindows, [bool]$MacOS = $IsMacOS ) $userHome = [Environment]::GetFolderPath('UserProfile') $fromHome = { param([string[]]$Parts) if (-not $userHome) { throw 'could not find home directory' } [IO.Path]::Join([string[]](@($userHome) + $Parts)) } $getVar = { param([string]$Name) [Environment]::GetEnvironmentVariable($Name) } $xdgOrLocal = { $xdg = & $getVar 'XDG_DATA_HOME' if ($null -ne $xdg) { $xdg } else { & $fromHome '.local', 'share' } } switch ($Source) { 'z' { $value = & $getVar '_Z_DATA' if ($null -ne $value) { , [string[]]@($value) } else { , [string[]]@(& $fromHome '.z') } } 'fasd' { $value = & $getVar '_FASD_DATA' if ($null -ne $value) { , [string[]]@($value) } else { , [string[]]@(& $fromHome '.fasd') } } 'zsh-z' { $value = & $getVar 'ZSHZ_DATA' if ($null -eq $value) { $value = & $getVar '_Z_DATA' } if ($null -ne $value) { , [string[]]@($value) } else { , [string[]]@(& $fromHome '.z') } } 'z.lua' { $value = & $getVar '_ZL_DATA' $usable = $value -and ($Windows -or $value -notmatch '^[A-Za-z]:[/\\]') $first = if ($usable) { $value } else { & $fromHome '.zlua' } , [string[]]@($first, [IO.Path]::Join((& $xdgOrLocal), 'zlua', 'zlua.txt')) } 'autojump' { $base = if ($MacOS) { & $fromHome 'Library' } elseif ($Windows) { $appData = & $getVar 'APPDATA' if ($null -eq $appData) { throw '%APPDATA% is not set' } $appData } else { & $xdgOrLocal } , [string[]]@([IO.Path]::Join($base, 'autojump', 'autojump.txt')) } } } function Read-ZoImportFile { <# .SYNOPSIS Reads an importer's data file: returns its Path and Bytes, or throws 'could not read <path>'. .DESCRIPTION Mirrors the File::open calls in the importers. Each candidate from Get-ZoImportDataPath is tried in order, moving on only when the file is not found (z.lua's fish fallback); when every candidate is missing, the error names the first one with its own cause. The error is `could not read {path:?}` (Format-ZoDebugString) with the OS error as the cause. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [string[]]$Path ) $firstError = $null foreach ($candidate in $Path) { try { $bytes = [IO.File]::ReadAllBytes($candidate) return [pscustomobject]@{ Path = $candidate; Bytes = $bytes } } catch { $exception = $_.Exception while ($exception -is [Management.Automation.MethodInvocationException] -and $null -ne $exception.InnerException) { $exception = $exception.InnerException } $failure = New-ZoError -Chain "could not read $(Format-ZoDebugString -Text $candidate)", (Get-ZoIoErrorText -Exception $exception) $notFound = $exception -is [IO.FileNotFoundException] -or $exception -is [IO.DirectoryNotFoundException] if (-not $notFound) { throw $failure } if ($null -eq $firstError) { $firstError = $failure } } } throw $firstError } function Split-ZoImportRecord { <# .SYNOPSIS Splits raw input into numbered, non-empty records. .DESCRIPTION Mirrors the Iterator impls in z.rs and autojump.rs (read_until '\n', then drop one '\n' and one '\r') and atuin.rs (read_until NUL, then drop the NUL). Every record read counts toward the 1-based line number, including the empty ones, which are skipped. Each record is then decoded as strict UTF-8, as each importer's next() does with `str::from_utf8(line).context("invalid utf-8")`. Outputs Number, Line (the decoded record without its terminator, or $null) and Error ($null, or 'invalid utf-8: <Utf8Error>' as anyhow's alternate format prints it) per record. .PARAMETER Separator 10 (LF, with the CR strip) or 0 (NUL). #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [byte[]]$Bytes, [ValidateSet(0, 10)] [byte]$Separator = 10 ) $start = 0 $number = 0 while ($start -lt $Bytes.Length) { $number++ $end = [Array]::IndexOf($Bytes, $Separator, $start) $next = if ($end -lt 0) { $Bytes.Length } else { $end + 1 } $length = if ($end -lt 0) { $Bytes.Length - $start } else { $end - $start } if ($Separator -eq 10 -and $length -gt 0 -and $Bytes[$start + $length - 1] -eq 13) { $length-- } if ($length -gt 0) { $slice = [byte[]]::new($length) [Array]::Copy($Bytes, $start, $slice, 0, $length) try { [pscustomobject]@{ Number = $number; Line = $script:ZoStrictUtf8.GetString($slice); Error = $null } } catch { [pscustomobject]@{ Number = $number; Line = $null; Error = 'invalid utf-8: ' + (Get-ZoUtf8Error -Bytes $slice -Offset 0 -Count $slice.Length) } } } $start = $next } } function ConvertFrom-ZoUInt64 { <# .SYNOPSIS Parses a u64 the way Rust's str::parse::<u64> does, or returns $null. .DESCRIPTION An optional '+', then one or more ASCII digits, no larger than u64::MAX. #> [CmdletBinding()] [OutputType([uint64])] param( [Parameter(Mandatory)] [AllowEmptyString()] [string]$Text ) if ($Text -cnotmatch '^\+?[0-9]+$') { return $null } $value = [uint64]0 if ([uint64]::TryParse($Text.TrimStart('+'), [Globalization.NumberStyles]::None, [Globalization.CultureInfo]::InvariantCulture, [ref]$value)) { return $value } $null } function Import-ZoZFormat { <# .SYNOPSIS Parses z's data file format (also fasd, zsh-z and z.lua) into records. .DESCRIPTION Mirrors z::Iter: each line is `path|rank|last_accessed`, split from the right at most twice (rsplitn(3, '|')) so a path may contain '|'. last_accessed is a u64, rank an f64 (Rust syntax). Any failure gives '<file>:<line>: invalid entry: <line>'; invalid UTF-8 gives '<file>:<line>: invalid utf-8: ...'. .PARAMETER Path The data file, as it appears in the error location. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [byte[]]$Bytes, [Parameter(Mandatory)] [string]$Path ) foreach ($record in Split-ZoImportRecord -Bytes $Bytes -Separator 10) { $location = "${Path}:$($record.Number)" if ($null -eq $record.Line) { [pscustomobject]@{ Error = "${location}: $($record.Error)" }; continue } $line = $record.Line $invalid = [pscustomobject]@{ Error = "${location}: invalid entry: $line" } $last = $line.LastIndexOf('|') if ($last -lt 0) { $invalid; continue } $lastAccessed = ConvertFrom-ZoUInt64 -Text $line.Substring($last + 1) if ($null -eq $lastAccessed) { $invalid; continue } $rest = $line.Substring(0, $last) $split = $rest.LastIndexOf('|') $rankText = if ($split -lt 0) { $rest } else { $rest.Substring($split + 1) } try { $rank = ConvertFrom-ZoFloat -Text $rankText } catch { $invalid; continue } if ($split -lt 0) { $invalid; continue } [pscustomobject]@{ Path = $rest.Substring(0, $split); Rank = $rank; LastAccessed = $lastAccessed } } } function Import-ZoAutojump { <# .SYNOPSIS Parses autojump's data file into records. .DESCRIPTION Mirrors autojump::Iter: each line is `rank<TAB>path` (split at the first tab). The rank is an f64 squashed with the sigmoid 1 / (1 + e^-x), because autojump scores differently, and last_accessed is 0. A line without a tab gives 'invalid entry: <line>'; a bad rank gives 'invalid rank: <rank>: <float error>'. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [byte[]]$Bytes, [Parameter(Mandatory)] [string]$Path ) foreach ($record in Split-ZoImportRecord -Bytes $Bytes -Separator 10) { $location = "${Path}:$($record.Number)" if ($null -eq $record.Line) { [pscustomobject]@{ Error = "${location}: $($record.Error)" }; continue } $line = $record.Line $tab = $line.IndexOf("`t") if ($tab -lt 0) { [pscustomobject]@{ Error = "${location}: invalid entry: $line" }; continue } $rankText = $line.Substring(0, $tab) try { $rank = ConvertFrom-ZoFloat -Text $rankText } catch { [pscustomobject]@{ Error = "${location}: invalid rank: ${rankText}: $($_.Exception.Message)" } continue } # sigmoid(x) = 1 / (1 + (-x).exp()). Rust's -x flips the sign bit, NaN included; PowerShell's # unary minus leaves a NaN's sign alone, and db.zo stores the bits, so flip it explicitly. $negated = [BitConverter]::Int64BitsToDouble([BitConverter]::DoubleToInt64Bits($rank) -bxor [long]::MinValue) [pscustomobject]@{ Path = $line.Substring($tab + 1); Rank = 1.0 / (1.0 + [Math]::Exp($negated)); LastAccessed = [uint64]0 } } } function Read-ZoAtuinHistory { <# .SYNOPSIS Runs `atuin history list` and returns its raw stdout bytes. .DESCRIPTION Mirrors the Command in atuin.rs: `atuin history list --format={time}<TAB>{directory} --print0`, stdout piped, stderr inherited. The program is resolved on PATH first (Find-ZoExecutable) and started by its full path. When it is missing, or fails to start, this throws 'failed to run `atuin`; is it installed and on PATH?' with the cause. .PARAMETER FilePath The program to run. Defaults to atuin on PATH. #> [CmdletBinding()] [OutputType([byte[]])] param( [string]$FilePath = (Find-ZoExecutable -Name 'atuin'), [string[]]$ArgumentList = @('history', 'list', "--format={time}`t{directory}", '--print0') ) $failed = 'failed to run `atuin`; is it installed and on PATH?' if (-not $FilePath) { $cause = if ($IsWindows) { 'program not found' } else { 'No such file or directory (os error 2)' } throw (New-ZoError -Chain $failed, $cause) } $info = [Diagnostics.ProcessStartInfo]::new($FilePath) foreach ($argument in $ArgumentList) { $info.ArgumentList.Add($argument) } $info.UseShellExecute = $false $info.RedirectStandardOutput = $true try { $process = [Diagnostics.Process]::Start($info) } catch { throw (New-ZoError -Chain $failed -ErrorRecord $_) } try { $buffer = [IO.MemoryStream]::new() $process.StandardOutput.BaseStream.CopyTo($buffer) $process.WaitForExit() , $buffer.ToArray() } finally { $process.Dispose() } } function Import-ZoAtuin { <# .SYNOPSIS Parses atuin's history output into records. .DESCRIPTION Mirrors atuin::Iter: NUL-separated `time<TAB>directory` records, the time in UTC as 'YYYY-MM-DD HH:MM:SS'. Each gives rank 1 and last_accessed = that time; a record whose directory equals the previous good record's is dropped (one entry per directory change). Errors are located as 'line <n>': 'invalid entry: <record>' without a tab, and 'invalid timestamp: "<time>": <reason>' for a bad time. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [byte[]]$Bytes ) $previous = $null foreach ($record in Split-ZoImportRecord -Bytes $Bytes -Separator 0) { $location = "line $($record.Number)" if ($null -eq $record.Line) { [pscustomobject]@{ Error = "${location}: $($record.Error)" }; continue } $line = $record.Line $tab = $line.IndexOf("`t") if ($tab -lt 0) { [pscustomobject]@{ Error = "${location}: invalid entry: $line" }; continue } $timeText = $line.Substring(0, $tab) $time = [DateTimeOffset]::MinValue $styles = [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal if (-not [DateTimeOffset]::TryParseExact($timeText, 'yyyy-MM-dd HH:mm:ss', [Globalization.CultureInfo]::InvariantCulture, $styles, [ref]$time)) { [pscustomobject]@{ Error = "${location}: invalid timestamp: $(Format-ZoDebugString -Text $timeText): the input does not match the format" } continue } $path = $line.Substring($tab + 1) if ($null -ne $previous -and $path -ceq $previous) { continue } $previous = $path # `timestamp as Epoch`: a time before 1970 wraps around, as the cast does. $seconds = $time.ToUnixTimeSeconds() [pscustomobject]@{ Path = $path; Rank = 1.0; LastAccessed = [BitConverter]::ToUInt64([BitConverter]::GetBytes($seconds), 0) } } } function Invoke-ZoImport { <# .SYNOPSIS Imports another tool's database into zoxide's. .DESCRIPTION Mirrors Import::run (cmd/import.rs) and import::run: without --merge a non-empty database throws 'current database is not empty, specify --merge to continue anyway'. Then the exclude globs are read, the source's records are parsed (each bad record is printed to stderr as '<location>: <reason>' and skipped), excluded paths are skipped, and the rest are appended with Database::add_unchecked. When anything was added the database is deduplicated and aged (_ZO_MAXAGE is read only then), and it is saved when dirty. .PARAMETER Subcommand The source: atuin, autojump, fasd, z, z.lua or zsh-z. #> [CmdletBinding()] param( [Parameter(Mandatory)] [hashtable]$Values, [Parameter(Mandatory)] [string]$Subcommand ) $database = Open-ZoDatabase if (-not $Values.merge -and $database.Dirs.Count -gt 0) { throw 'current database is not empty, specify --merge to continue anyway' } $exclude = @(Get-ZoExcludeDirs) $records = if ($Subcommand -eq 'atuin') { Import-ZoAtuin -Bytes (Read-ZoAtuinHistory) } else { $file = Read-ZoImportFile -Path (Get-ZoImportDataPath -Source $Subcommand) if ($Subcommand -eq 'autojump') { Import-ZoAutojump -Bytes $file.Bytes -Path $file.Path } else { Import-ZoZFormat -Bytes $file.Bytes -Path $file.Path } } foreach ($record in $records) { if ($record.PSObject.Properties['Error']) { Write-ZoError -Message $record.Error continue } $excluded = $false foreach ($glob in $exclude) { if (Test-ZoGlobMatch -Pattern $glob -Path $record.Path) { $excluded = $true; break } } if ($excluded) { continue } Add-ZoDirUnchecked -Database $database -Path $record.Path -Rank $record.Rank -Now $record.LastAccessed } if ($database.Dirty) { Invoke-ZoDedup -Database $database Invoke-ZoAge -Database $database -MaxAge (Get-ZoMaxAge) } Save-ZoDatabase -Database $database } |