From bff8b81d596c5e7c73a3327a3af596bd3d7bc875 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Tue, 16 Jun 2026 10:44:44 +0300 Subject: [PATCH 1/4] Update version to 0.1.7 in package.json, enhance installation scripts for better PATH management, and improve README instructions for setup. The install scripts now auto-configure the user's PATH and provide clearer next steps after installation. --- README.md | 2 +- docs/install.ps1 | 165 +++++++++++++++++++++++++++++++++++------------ docs/install.sh | 162 +++++++++++++++++++++++++++++++++++++++------- package.json | 2 +- 4 files changed, 263 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 79af555..87aa1b6 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ Run `decodo setup` or export `DECODO_AUTH_TOKEN`. **`command not found: decodo`** -Ensure npm's global bin directory is on your `PATH` after `npm install -g`. Re-run the [install script](https://decodo.github.io/cli/install.sh) or use `npx @decodo/cli`. +The [install script](https://decodo.github.io/cli/install.sh) auto-configures PATH and prints a `source` step — re-run it or open a new terminal. You can also use `npx @decodo/cli`. **Validation / API errors** diff --git a/docs/install.ps1 b/docs/install.ps1 index 7a99f3f..2adbe70 100644 --- a/docs/install.ps1 +++ b/docs/install.ps1 @@ -5,6 +5,10 @@ $PackageName = '@decodo/cli' $CommandName = 'decodo' $MinNodeMajor = 18 +$script:OrigPath = $env:PATH +$script:UserPrefix = $null +$script:PathActivationRequired = $false + function Write-Info([string]$Message) { Write-Host "==> $Message" -ForegroundColor Blue } @@ -18,6 +22,15 @@ function Write-Err([string]$Message) { exit 1 } +function Write-Ok([string]$Message) { + Write-Host $Message -ForegroundColor Green +} + +function Test-PathContains([string]$Dir, [string]$PathValue) { + $entries = $PathValue -split ';' | Where-Object { $_ -ne '' } + return $entries -contains $Dir +} + function Get-NodeVersion { if (-not (Get-Command node -ErrorAction SilentlyContinue)) { Write-Err @" @@ -39,20 +52,6 @@ Update Node.js from https://nodejs.org/ and try again. return $version } -Write-Host '' -Write-Host 'Decodo CLI Installer' -ForegroundColor White -Write-Host '' - -$nodeVersion = Get-NodeVersion -Write-Info "Found Node.js v$nodeVersion" - -if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { - Write-Err 'npm is not available. Install npm and try again.' -} - -$UserPrefix = $null - -# Run npm install without aborting on failure, so we can fall back. Returns the exit code. function Invoke-NpmInstall { param([string[]]$NpmArgs) try { @@ -83,44 +82,124 @@ Try fixing your npm permissions, or run the CLI without installing: npx $Package $env:PATH = "$script:UserPrefix;$env:PATH" } -Install-Package +function Resolve-InstallBin { + if ($script:UserPrefix) { + return $script:UserPrefix + } -$installedVersion = $null -if (Get-Command $CommandName -ErrorAction SilentlyContinue) { - $installedVersion = & $CommandName --version 2>$null + $npmPrefix = (npm prefix -g 2>$null).Trim() + if (-not $npmPrefix) { + Write-Err 'Could not determine npm global bin directory.' + } + + return $npmPrefix } -if ($installedVersion) { - Write-Host '' - Write-Host "Success! $PackageName $installedVersion is installed." -ForegroundColor Green -} else { - Write-Host '' - Write-Host 'Installed! You may need to restart your shell or add the npm global bin directory to your PATH.' -ForegroundColor Green - - if (-not $UserPrefix) { - $npmPrefix = (npm prefix -g 2>$null).Trim() - if ($npmPrefix) { - $pathEntries = $env:PATH -split ';' | Where-Object { $_ -ne '' } - if ($pathEntries -notcontains $npmPrefix) { - Write-Warn "$npmPrefix is not in your PATH. Add it with:" - Write-Host " setx PATH `"$npmPrefix;%PATH%`"" - Write-Host '' - } +function Ensure-InstallPath([string]$BinDir) { + if (Test-PathContains $BinDir $script:OrigPath) { + return + } + + $script:PathActivationRequired = $true + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + + if ($userPath -and $userPath -like "*$BinDir*") { + Write-Warn "$BinDir is in your user PATH but not active in this shell." + Write-Warn 'Restart this terminal, then run decodo setup.' + return + } + + $newPath = if ($userPath) { "$BinDir;$userPath" } else { $BinDir } + [Environment]::SetEnvironmentVariable('Path', $newPath, 'User') + Write-Warn "$BinDir was not in your PATH. Added it to your user PATH." + Write-Warn 'Restart this terminal, then run decodo setup.' +} + +function Get-DecodoBin([string]$BinDir) { + $cmd = Join-Path $BinDir 'decodo.cmd' + if (Test-Path $cmd) { + return $cmd + } + + return Join-Path $BinDir 'decodo' +} + +function Get-CommandPrefix([string]$BinDir) { + if ($script:PathActivationRequired) { + return Get-DecodoBin $BinDir + } + + return $CommandName +} + +function Offer-Setup([string]$BinDir) { + $decodoBin = Get-DecodoBin $BinDir + if (-not (Test-Path $decodoBin)) { + Write-Err "Could not find $decodoBin after install." + } + + if (-not [Console]::IsInputRedirected -and -not [Console]::IsOutputRedirected) { + Write-Host '' + Write-Host 'Next: configure your auth token.' + Write-Host '' + $answer = Read-Host 'Continue with setup? [Y/n]' + if ($answer -match '^[nN]') { + $cmd = Get-CommandPrefix $BinDir + Write-Host '' + Write-Host "Run $cmd setup when you are ready." + Write-Host '' + return } + + & $decodoBin setup + return } + + $cmd = Get-CommandPrefix $BinDir + Write-Host '' + Write-Host "Next step: configure your auth token with $cmd setup" + Write-Host '' } -if ($UserPrefix) { +function Print-NextSteps([string]$BinDir) { + $cmd = Get-CommandPrefix $BinDir + Write-Host 'Get started:' + Write-Host " $cmd scrape https://ip.decodo.com" + Write-Host ' $cmd search "decodo scraping api"' + Write-Host " $cmd whoami" Write-Host '' - Write-Host "The CLI was installed to $UserPrefix." - Write-Host 'Add it to your PATH permanently with:' - Write-Host " setx PATH `"$UserPrefix;%PATH%`"" } Write-Host '' -Write-Host 'Next step: configure your auth token with decodo setup' -Write-Host 'Get started:' -Write-Host ' decodo scrape https://ip.decodo.com' -Write-Host ' decodo search "decodo scraping api"' -Write-Host ' decodo whoami' +Write-Host 'Decodo CLI Installer' -ForegroundColor White +Write-Host '' + +$nodeVersion = Get-NodeVersion +Write-Info "Found Node.js v$nodeVersion" + +if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + Write-Err 'npm is not available. Install npm and try again.' +} + +Install-Package + +$binDir = Resolve-InstallBin +Ensure-InstallPath $binDir + +$decodoBin = Get-DecodoBin $binDir +$installedVersion = & $decodoBin --version 2>$null +if (-not $installedVersion) { + $installedVersion = 'unknown' +} + Write-Host '' +Write-Ok "Success! $PackageName $installedVersion is installed." + +if ($script:PathActivationRequired) { + Write-Host '' +} else { + Write-Ok 'Ready to use — decodo is on your PATH.' +} + +Offer-Setup $binDir +Print-NextSteps $binDir diff --git a/docs/install.sh b/docs/install.sh index 1eda9d8..24c5505 100755 --- a/docs/install.sh +++ b/docs/install.sh @@ -5,6 +5,11 @@ PACKAGE_NAME="@decodo/cli" COMMAND_NAME="decodo" MIN_NODE_MAJOR=18 +ORIG_PATH="" +USER_PREFIX_BIN="" +PATH_ACTIVATION_REQUIRED=0 +ACTIVATION_RC_FILE="" + if [ -t 1 ]; then RED='\033[0;31m' GREEN='\033[0;32m' @@ -19,8 +24,128 @@ fi info() { printf "${BLUE}${BOLD}==>${RESET} %s\n" "$1"; } warn() { printf "${YELLOW}${BOLD}warning:${RESET} %s\n" "$1"; } +success() { printf "${GREEN}${BOLD}%s${RESET}\n" "$1"; } error() { printf "${RED}${BOLD}error:${RESET} %s\n" "$1" >&2; exit 1; } +path_contains() { + dir="$1" + path_list="$2" + case ":${path_list}:" in + *":${dir}:"*) return 0 ;; + esac + return 1 +} + +shell_rc_file() { + shell_name=$(basename "${SHELL:-/bin/sh}") + case "$shell_name" in + zsh) printf '%s' "$HOME/.zshrc" ;; + bash) printf '%s' "$HOME/.bashrc" ;; + fish) printf '%s' "$HOME/.config/fish/config.fish" ;; + *) printf '%s' "$HOME/.profile" ;; + esac +} + +resolve_install_bin() { + if [ -n "$USER_PREFIX_BIN" ]; then + printf '%s' "$USER_PREFIX_BIN" + return 0 + fi + npm_prefix=$(npm prefix -g 2>/dev/null) || error "Could not determine npm global bin directory." + printf '%s/bin' "$npm_prefix" +} + +ensure_path() { + dir="$1" + + if path_contains "$dir" "$ORIG_PATH"; then + return 0 + fi + + rc_file=$(shell_rc_file) + ACTIVATION_RC_FILE="$rc_file" + PATH_ACTIVATION_REQUIRED=1 + + if [ -f "$rc_file" ] && grep -Fq "$dir" "$rc_file" 2>/dev/null; then + warn "$dir is in $rc_file but not active in this shell." + return 0 + fi + + mkdir -p "$(dirname "$rc_file")" + printf '\n' >> "$rc_file" + shell_name=$(basename "${SHELL:-/bin/sh}") + if [ "$shell_name" = "fish" ]; then + printf 'set -gx PATH "%s" $PATH\n' "$dir" >> "$rc_file" + else + printf 'export PATH="%s:$PATH"\n' "$dir" >> "$rc_file" + fi + + warn "$dir was not in your PATH. Added it to $rc_file" +} + +command_prefix() { + bin_dir="$1" + if [ "$PATH_ACTIVATION_REQUIRED" = 1 ]; then + printf '%s/%s' "$bin_dir" "$COMMAND_NAME" + else + printf '%s' "$COMMAND_NAME" + fi +} + +print_activation_steps() { + bin_dir="$1" + if [ "$PATH_ACTIVATION_REQUIRED" != 1 ]; then + return 0 + fi + printf '\n' + warn "Run this now: source ${ACTIVATION_RC_FILE}" + warn "Or: export PATH=\"${bin_dir}:\$PATH\"" +} + +offer_setup() { + bin_dir="$1" + decodo_bin="${bin_dir}/${COMMAND_NAME}" + + if ! [ -f "$decodo_bin" ]; then + error "Could not find ${decodo_bin} after install." + fi + + if ! [ -t 0 ] || ! [ -t 1 ]; then + cmd=$(command_prefix "$bin_dir") + printf '\nNext step: configure your auth token with %s%s setup%s\n\n' "$BOLD" "$cmd" "$RESET" + return 0 + fi + + printf '\nNext: configure your auth token.\n\n' + printf 'Continue with setup? [Y/n] ' + if ! read -r answer /dev/null; then + cmd=$(command_prefix "$bin_dir") + printf '\nRun %s setup when you are ready.\n\n' "$cmd" + return 0 + fi + printf '\n' + + case "$answer" in + [nN]*) + cmd=$(command_prefix "$bin_dir") + printf 'Run %s setup when you are ready.\n\n' "$cmd" + return 0 + ;; + esac + + "$decodo_bin" setup +} + +print_next_steps() { + bin_dir="$1" + cmd=$(command_prefix "$bin_dir") + + printf 'Get started:\n' + printf ' %s%s scrape%s https://ip.decodo.com\n' "$BOLD" "$cmd" "$RESET" + printf ' %s%s search%s "decodo scraping api"\n' "$BOLD" "$cmd" "$RESET" + printf ' %s%s whoami%s\n\n' "$BOLD" "$cmd" "$RESET" +} + check_platform() { case "$(uname -s)" in Linux|Darwin) ;; @@ -92,6 +217,8 @@ or run the CLI without installing: npx ${PACKAGE_NAME} --help" main() { printf "\n${BOLD}Decodo CLI Installer${RESET}\n\n" + ORIG_PATH="$PATH" + check_platform NODE_VERSION=$(check_node) info "Found Node.js v${NODE_VERSION}" @@ -102,32 +229,21 @@ main() { install_package - if command -v "$COMMAND_NAME" >/dev/null 2>&1; then - installed_version=$("$COMMAND_NAME" --version 2>/dev/null || echo "unknown") - printf "\n${GREEN}${BOLD}Success!${RESET} ${PACKAGE_NAME} ${installed_version} is installed.\n" - else - printf "\n${GREEN}${BOLD}Installed!${RESET} You may need to restart your shell or add the npm global bin directory to your PATH.\n" - if [ -z "$USER_PREFIX_BIN" ]; then - npm_prefix=$(npm config get prefix 2>/dev/null) || true - npm_bin="${npm_prefix:+$npm_prefix/bin}" - if [ -n "$npm_bin" ] && ! echo "$PATH" | tr ':' '\n' | grep -qx "$npm_bin"; then - warn "${npm_bin} is not in your PATH. Add it with:" - printf " export PATH=\"%s:\$PATH\"\n\n" "$npm_bin" - fi - fi - fi + BIN_DIR=$(resolve_install_bin) + ensure_path "$BIN_DIR" - if [ -n "$USER_PREFIX_BIN" ]; then - printf "\nThe CLI was installed to ${BOLD}%s${RESET}.\n" "$USER_PREFIX_BIN" - printf "Add it to your PATH permanently by appending this line to your shell profile (e.g. ~/.zshrc or ~/.bashrc):\n" - printf " ${BOLD}export PATH=\"%s:\$PATH\"${RESET}\n" "$USER_PREFIX_BIN" + installed_version=$("${BIN_DIR}/${COMMAND_NAME}" --version 2>/dev/null || echo "unknown") + printf '\n' + success "Success! ${PACKAGE_NAME} ${installed_version} is installed." + + if [ "$PATH_ACTIVATION_REQUIRED" = 1 ]; then + print_activation_steps "$BIN_DIR" + else + success "Ready to use — decodo is on your PATH." fi - printf "\nNext step: configure your auth token with ${BOLD}decodo setup${RESET}\n" - printf "Get started:\n" - printf " ${BOLD}decodo scrape${RESET} https://ip.decodo.com\n" - printf " ${BOLD}decodo search${RESET} \"decodo scraping api\"\n" - printf " ${BOLD}decodo whoami${RESET}\n\n" + offer_setup "$BIN_DIR" + print_next_steps "$BIN_DIR" } main diff --git a/package.json b/package.json index 4544126..7d0f188 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@decodo/cli", - "version": "0.1.6", + "version": "0.1.7", "description": "Official CLI for the Decodo APIs", "license": "MIT", "type": "module", From 8750833ec83e661f5c91c08dc6666093b7b1c080 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Tue, 16 Jun 2026 10:55:53 +0300 Subject: [PATCH 2/4] Refactor install.sh to enhance output formatting for next steps. Updated printf statements to use consistent syntax for better readability and maintainability. --- docs/install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/install.sh b/docs/install.sh index 24c5505..e178995 100755 --- a/docs/install.sh +++ b/docs/install.sh @@ -112,7 +112,7 @@ offer_setup() { if ! [ -t 0 ] || ! [ -t 1 ]; then cmd=$(command_prefix "$bin_dir") - printf '\nNext step: configure your auth token with %s%s setup%s\n\n' "$BOLD" "$cmd" "$RESET" + printf "\nNext step: configure your auth token with ${BOLD}%s setup${RESET}\n\n" "$cmd" return 0 fi @@ -141,9 +141,9 @@ print_next_steps() { cmd=$(command_prefix "$bin_dir") printf 'Get started:\n' - printf ' %s%s scrape%s https://ip.decodo.com\n' "$BOLD" "$cmd" "$RESET" - printf ' %s%s search%s "decodo scraping api"\n' "$BOLD" "$cmd" "$RESET" - printf ' %s%s whoami%s\n\n' "$BOLD" "$cmd" "$RESET" + printf " ${BOLD}%s scrape${RESET} https://ip.decodo.com\n" "$cmd" + printf " ${BOLD}%s search${RESET} \"decodo scraping api\"\n" "$cmd" + printf " ${BOLD}%s whoami${RESET}\n\n" "$cmd" } check_platform() { From 92d8080f2ed6fbf3c33ecf83f80e8918cb833821 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Tue, 16 Jun 2026 11:01:21 +0300 Subject: [PATCH 3/4] Fix output formatting in install.ps1 for next steps command. Updated string interpolation for improved clarity in user instructions. --- docs/install.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.ps1 b/docs/install.ps1 index 2adbe70..e0651ec 100644 --- a/docs/install.ps1 +++ b/docs/install.ps1 @@ -165,7 +165,7 @@ function Print-NextSteps([string]$BinDir) { $cmd = Get-CommandPrefix $BinDir Write-Host 'Get started:' Write-Host " $cmd scrape https://ip.decodo.com" - Write-Host ' $cmd search "decodo scraping api"' + Write-Host " $cmd search ""decodo scraping api""" Write-Host " $cmd whoami" Write-Host '' } From d06c22fe6a739534fb864c6ab28a0f269f568fe0 Mon Sep 17 00:00:00 2001 From: paulius-krutkis-dcd Date: Wed, 17 Jun 2026 11:01:56 +0300 Subject: [PATCH 4/4] Fix critical audit issues and improve install PATH setup (#36) * Fix remaining critical CLI audit items - Object/array schema params now register as --flag with a JSON-parsing argParser; invalid JSON is a usage error - Derive binary PNG output context from the built body so generated commands (universal --headless png) write a PNG, not base64 text - Map syscall-coded network failures (ENOTFOUND, ECONNREFUSED, ...) in the cause chain to exit 7 and render the cause - setup now rejects as a usage error on EOF/closed stdin instead of exiting 0 having saved nothing; handle TTY EOF and Ctrl+D - Add offline README docs regression test validating documented commands and flags against the command tree built from BundledSchema * Skip parse/markdown defaults for headless png screenshots Manual end-to-end testing surfaced that generated commands routed PNG output correctly but applyRequestDefaults still injected markdown:true, so the API returned markdown instead of a screenshot and PNG extraction failed. Skip the defaults when headless is png so universal --headless png writes a real PNG, matching the curated screenshot command. * Refactor CLI error handling by introducing CliUsageError class - Moved CliUsageError definition to a new file for better organization. - Updated imports across multiple files to reference the new location of CliUsageError. - Added tests for CliUsageError to ensure proper functionality and error message handling. --- src/auth/commands/setup.ts | 6 +- src/output/services/apply-request-defaults.ts | 4 + src/platform/errors/cli-usage-error.ts | 6 + src/platform/services/handle-cli-error.ts | 41 +++- src/platform/services/prompt-hidden.ts | 48 ++++- src/platform/services/write-binary.ts | 3 +- src/scrape/services/command-builder.ts | 27 ++- src/scrape/services/run-target-scrape.ts | 23 ++- tests/docs/readme-commands.test.ts | 184 ++++++++++++++++++ .../services/apply-request-defaults.test.ts | 16 ++ tests/platform/errors/cli-usage-error.test.ts | 12 ++ .../services/handle-cli-error.test.ts | 22 ++- tests/platform/services/prompt-hidden.test.ts | 27 ++- tests/scrape/services/command-builder.test.ts | 42 ++++ .../scrape/services/run-target-scrape.test.ts | 45 +++++ 15 files changed, 482 insertions(+), 24 deletions(-) create mode 100644 src/platform/errors/cli-usage-error.ts create mode 100644 tests/docs/readme-commands.test.ts create mode 100644 tests/platform/errors/cli-usage-error.test.ts diff --git a/src/auth/commands/setup.ts b/src/auth/commands/setup.ts index 213a0d2..015f0ca 100644 --- a/src/auth/commands/setup.ts +++ b/src/auth/commands/setup.ts @@ -1,9 +1,7 @@ import { Command } from "commander"; import { getRootOpts } from "../../cli/services/global-opts.js"; -import { - CliUsageError, - handleCliError, -} from "../../platform/services/handle-cli-error.js"; +import { CliUsageError } from "../../platform/errors/cli-usage-error.js"; +import { handleCliError } from "../../platform/services/handle-cli-error.js"; import { promptHidden } from "../../platform/services/prompt-hidden.js"; import { validateAuthToken } from "../../scrape/services/auth-validation.js"; import { PLAYGROUND_URL } from "../constants.js"; diff --git a/src/output/services/apply-request-defaults.ts b/src/output/services/apply-request-defaults.ts index b114468..44c78e0 100644 --- a/src/output/services/apply-request-defaults.ts +++ b/src/output/services/apply-request-defaults.ts @@ -5,6 +5,10 @@ export function applyRequestDefaults( target: string, schema: DecodoSchema ): void { + if (body.headless === "png") { + return; + } + const properties = schema.getTargetParameterSchema(target)?.properties ?? {}; if (properties.parse !== undefined && body.parse === undefined) { diff --git a/src/platform/errors/cli-usage-error.ts b/src/platform/errors/cli-usage-error.ts new file mode 100644 index 0000000..d58e96e --- /dev/null +++ b/src/platform/errors/cli-usage-error.ts @@ -0,0 +1,6 @@ +export class CliUsageError extends Error { + constructor(message: string) { + super(message); + this.name = "CliUsageError"; + } +} diff --git a/src/platform/services/handle-cli-error.ts b/src/platform/services/handle-cli-error.ts index 7de14f3..a5789dd 100644 --- a/src/platform/services/handle-cli-error.ts +++ b/src/platform/services/handle-cli-error.ts @@ -8,14 +8,38 @@ import { import { PLAYGROUND_URL } from "../../auth/constants.js"; import { AuthRequiredError } from "../../auth/errors/auth-required-error.js"; import { EXIT } from "../constants.js"; +import { CliUsageError } from "../errors/cli-usage-error.js"; const EXIT_SIGNAL_PREFIX = "process.exit:"; -export class CliUsageError extends Error { - constructor(message: string) { - super(message); - this.name = "CliUsageError"; +const NETWORK_ERROR_CODES = new Set([ + "ENOTFOUND", + "ECONNREFUSED", + "ECONNRESET", + "ETIMEDOUT", + "EAI_AGAIN", + "EHOSTUNREACH", + "ENETUNREACH", + "EPIPE", +]); + +function findNetworkCause( + err: unknown +): { code: string; message: string } | undefined { + const seen = new Set(); + let current: unknown = err; + + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + const code = (current as { code?: unknown }).code; + if (typeof code === "string" && NETWORK_ERROR_CODES.has(code)) { + const message = (current as { message?: unknown }).message; + return { code, message: typeof message === "string" ? message : code }; + } + current = (current as { cause?: unknown }).cause; } + + return; } export function resolveCliExitCode(err: unknown): number { @@ -43,6 +67,10 @@ export function resolveCliExitCode(err: unknown): number { return EXIT.NETWORK; } + if (findNetworkCause(err)) { + return EXIT.NETWORK; + } + return EXIT.ERROR; } @@ -104,6 +132,11 @@ export function handleCliError( console.error(`Error: ${message}`); + const networkCause = findNetworkCause(err); + if (networkCause) { + console.error(`Cause: ${networkCause.code} (${networkCause.message})`); + } + if (err instanceof ValidationError) { const details = extractValidationDetails(err); if (details.length > 0) { diff --git a/src/platform/services/prompt-hidden.ts b/src/platform/services/prompt-hidden.ts index 8297f52..8df4481 100644 --- a/src/platform/services/prompt-hidden.ts +++ b/src/platform/services/prompt-hidden.ts @@ -1,5 +1,11 @@ import { stdin, stdout } from "node:process"; import { createInterface } from "node:readline/promises"; +import { CliUsageError } from "../errors/cli-usage-error.js"; + +const CHAR_ETX = 3; +const CHAR_EOT = 4; +const CHAR_DEL = 127; +const CHAR_BACKSPACE = 8; interface HiddenPromptState { cleanup: () => void; @@ -9,13 +15,22 @@ interface HiddenPromptState { } function handleHiddenPromptChar(char: string, state: HiddenPromptState): void { - if (char === "\u0003") { + const code = char.charCodeAt(0); + + if (code === CHAR_ETX) { state.cleanup(); stdout.write("\n"); state.reject(new Error("Cancelled.")); return; } + if (code === CHAR_EOT) { + state.cleanup(); + stdout.write("\n"); + state.reject(new CliUsageError("No auth token provided on stdin.")); + return; + } + if (char === "\r" || char === "\n") { state.cleanup(); stdout.write("\n"); @@ -23,7 +38,7 @@ function handleHiddenPromptChar(char: string, state: HiddenPromptState): void { return; } - if (char === "\u007f" || char === "\b") { + if (code === CHAR_DEL || code === CHAR_BACKSPACE) { if (state.input.length > 0) { state.input = state.input.slice(0, -1); stdout.write("\b \b"); @@ -34,14 +49,23 @@ function handleHiddenPromptChar(char: string, state: HiddenPromptState): void { state.input += char; } +async function promptViaReadline(message: string): Promise { + const rl = createInterface({ input: stdin, output: stdout }); + try { + return await new Promise((resolve, reject) => { + rl.question(message).then((answer) => resolve(answer.trim()), reject); + rl.once("close", () => { + reject(new CliUsageError("No auth token provided on stdin.")); + }); + }); + } finally { + rl.close(); + } +} + export async function promptHidden(message: string): Promise { if (!stdin.isTTY) { - const rl = createInterface({ input: stdin, output: stdout }); - try { - return (await rl.question(message)).trim(); - } finally { - rl.close(); - } + return await promptViaReadline(message); } stdout.write(message); @@ -64,12 +88,20 @@ export async function promptHidden(message: string): Promise { } }; + const onEnd = (): void => { + state.cleanup(); + stdout.write("\n"); + reject(new CliUsageError("No auth token provided on stdin.")); + }; + state.cleanup = (): void => { stdin.setRawMode(false); stdin.pause(); stdin.removeListener("data", onData); + stdin.removeListener("end", onEnd); }; stdin.on("data", onData); + stdin.on("end", onEnd); }); } diff --git a/src/platform/services/write-binary.ts b/src/platform/services/write-binary.ts index 21787e7..d9104ea 100644 --- a/src/platform/services/write-binary.ts +++ b/src/platform/services/write-binary.ts @@ -1,6 +1,7 @@ import { writeFileSync } from "node:fs"; import { resolve } from "node:path"; -import { CliUsageError, handleCliError } from "./handle-cli-error.js"; +import { CliUsageError } from "../errors/cli-usage-error.js"; +import { handleCliError } from "./handle-cli-error.js"; import { resolveOutputFilePath } from "./resolve-output-file.js"; export const BINARY_TTY_ERROR = diff --git a/src/scrape/services/command-builder.ts b/src/scrape/services/command-builder.ts index f80bcf9..8d39f4f 100644 --- a/src/scrape/services/command-builder.ts +++ b/src/scrape/services/command-builder.ts @@ -1,5 +1,5 @@ import type { DecodoSchema } from "@decodo/sdk-ts"; -import { type Command, Option } from "commander"; +import { type Command, InvalidArgumentError, Option } from "commander"; import type { JSONSchema4 } from "json-schema"; import { attachScrapeOutputOptions } from "../../output/commands/attach-output-options.js"; import { applyRequestDefaults } from "../../output/services/apply-request-defaults.js"; @@ -25,9 +25,29 @@ function formatOptionHelp(propertySchema: JSONSchema4): string { return `${propertySchema.type}${bounds}`; } + if (propertySchema.type === "array") { + return "JSON array"; + } + + if (propertySchema.type === "object") { + return "JSON object"; + } + return String(propertySchema.type ?? "value"); } +function parseJsonArg(field: string) { + return (value: string): unknown => { + try { + return JSON.parse(value); + } catch { + throw new InvalidArgumentError( + `--${snakeToKebab(field)} expects valid JSON.` + ); + } + }; +} + function addPropertyOption( command: Command, field: string, @@ -64,6 +84,11 @@ function addPropertyOption( return; } + if (propertySchema.type === "array" || propertySchema.type === "object") { + command.option(`--${kebabFlag} `, help, parseJsonArg(field)); + return; + } + command.option(`--${kebabFlag} `, help); } diff --git a/src/scrape/services/run-target-scrape.ts b/src/scrape/services/run-target-scrape.ts index 683d639..50c2a04 100644 --- a/src/scrape/services/run-target-scrape.ts +++ b/src/scrape/services/run-target-scrape.ts @@ -6,6 +6,7 @@ import { getRootOpts } from "../../cli/services/global-opts.js"; import { verboseLog } from "../../cli/services/verbose-log.js"; import { writeScrapeResponse } from "../../output/services/write-scrape-response.js"; import type { OutputOptions } from "../../output/types/output-options.js"; +import type { WriteScrapeResponseContext } from "../../output/types/write-scrape-response.js"; import { handleCliError } from "../../platform/services/handle-cli-error.js"; import type { ExecuteScrapeOptions, @@ -39,6 +40,22 @@ async function executeScrape({ }); } +function resolveOutputContext( + explicit: Partial | undefined, + body: Record, + input: string | undefined +): Partial | undefined { + if (explicit?.binary) { + return explicit; + } + + if (body.headless === "png") { + return { ...explicit, binary: { kind: "png" }, input }; + } + + return explicit; +} + export function createTargetAction( target: string, schema: DecodoSchema, @@ -79,7 +96,11 @@ export function createTargetAction( const body = resolveBody(input, options); verboseLog(verbose, formatScrapeRequestLog(body)); - const outputContext = getOutputContext?.(input, options); + const outputContext = resolveOutputContext( + getOutputContext?.(input, options), + body, + input + ); await executeScrape({ token: auth.token, schema, diff --git a/tests/docs/readme-commands.test.ts b/tests/docs/readme-commands.test.ts new file mode 100644 index 0000000..dd8d361 --- /dev/null +++ b/tests/docs/readme-commands.test.ts @@ -0,0 +1,184 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { BundledSchema } from "@decodo/sdk-ts"; +import { Command } from "commander"; +import { describe, expect, it } from "vitest"; +import { resetCommand } from "../../src/auth/commands/reset.js"; +import { setupCommand } from "../../src/auth/commands/setup.js"; +import { whoamiCommand } from "../../src/auth/commands/whoami.js"; +import { createCodegenTargetCommands } from "../../src/scrape/commands/codegen-target-commands.js"; +import { createListTargetsCommand } from "../../src/scrape/commands/list-targets.js"; +import { createScrapeCommand } from "../../src/scrape/commands/scrape.js"; +import { createScreenshotCommand } from "../../src/scrape/commands/screenshot.js"; +import { createSearchCommand } from "../../src/scrape/commands/search.js"; + +const README_PATH = fileURLToPath(new URL("../../README.md", import.meta.url)); + +const GLOBAL_FLAGS = new Set([ + "-h", + "--help", + "-V", + "--version", + "-v", + "--verbose", + "--token", +]); + +const SHELL_BREAK = new Set(["|", "&&", "||", ";", "&", ">", ">>", "<"]); + +function buildProgram(): Command { + const schema = BundledSchema.shared; + const program = new Command("decodo") + .version("0.0.0", "-V, --version") + .option("-v, --verbose") + .option("--token "); + + for (const command of [ + setupCommand, + resetCommand, + whoamiCommand, + createScrapeCommand(schema), + createSearchCommand(schema), + createScreenshotCommand(schema), + createListTargetsCommand(schema), + ...createCodegenTargetCommands(schema), + ]) { + program.addCommand(command); + } + + return program; +} + +function extractFencedBlocks(markdown: string): string[] { + const blocks: string[] = []; + const fence = /```[^\n]*\n([\s\S]*?)```/g; + let match = fence.exec(markdown); + while (match) { + blocks.push(match[1]); + match = fence.exec(markdown); + } + return blocks; +} + +function tokenize(line: string): string[] { + const tokens: string[] = []; + const token = /"([^"]*)"|'([^']*)'|(\S+)/g; + let match = token.exec(line); + while (match) { + tokens.push(match[1] ?? match[2] ?? match[3] ?? ""); + match = token.exec(line); + } + return tokens; +} + +function extractInvocations(tokens: string[]): string[][] { + const invocations: string[][] = []; + let current: string[] | null = null; + + for (const tok of tokens) { + if (tok === "decodo") { + if (current) { + invocations.push(current); + } + current = []; + continue; + } + + if (!current) { + continue; + } + + if (SHELL_BREAK.has(tok)) { + invocations.push(current); + current = null; + continue; + } + + current.push(tok); + } + + if (current) { + invocations.push(current); + } + + return invocations; +} + +function collectInvocations(markdown: string): string[][] { + const invocations: string[][] = []; + for (const block of extractFencedBlocks(markdown)) { + for (const line of block.split("\n")) { + invocations.push(...extractInvocations(tokenize(line))); + } + } + return invocations; +} + +function knownFlags(command: Command, program: Command): Set { + const flags = new Set(GLOBAL_FLAGS); + for (const option of [...program.options, ...command.options]) { + if (option.long) { + flags.add(option.long); + } + if (option.short) { + flags.add(option.short); + } + } + return flags; +} + +function resolveSubcommand( + args: string[], + program: Command +): { command: Command; name: string } | undefined { + const name = args.find((arg) => !arg.startsWith("-")); + if (!name) { + return { command: program, name: "decodo" }; + } + + const command = program.commands.find( + (cmd) => cmd.name() === name || cmd.aliases().includes(name) + ); + return command ? { command, name } : undefined; +} + +describe("README command examples", () => { + const markdown = readFileSync(README_PATH, "utf8"); + const program = buildProgram(); + const invocations = collectInvocations(markdown); + + it("documents at least one decodo command", () => { + expect(invocations.length).toBeGreaterThan(0); + }); + + it("only references commands that exist in the offline command tree", () => { + const unknown = invocations.filter( + (args) => !resolveSubcommand(args, program) + ); + expect(unknown).toEqual([]); + }); + + it("only uses flags that exist on the referenced command", () => { + const violations: string[] = []; + + for (const args of invocations) { + const resolved = resolveSubcommand(args, program); + if (!resolved) { + continue; + } + + const flags = knownFlags(resolved.command, program); + for (const arg of args) { + if (!arg.startsWith("-")) { + continue; + } + const flag = arg.split("=")[0]; + if (!flags.has(flag)) { + violations.push(`${resolved.name}: ${flag}`); + } + } + } + + expect(violations).toEqual([]); + }); +}); diff --git a/tests/output/services/apply-request-defaults.test.ts b/tests/output/services/apply-request-defaults.test.ts index 9c49871..f02cc14 100644 --- a/tests/output/services/apply-request-defaults.test.ts +++ b/tests/output/services/apply-request-defaults.test.ts @@ -36,6 +36,22 @@ describe("applyRequestDefaults", () => { }); }); + it("skips parse/markdown defaults for headless png screenshots", () => { + const body: Record = { + target: Target.Universal, + url: "https://example.com", + headless: "png", + }; + + applyRequestDefaults(body, Target.Universal, schema); + + expect(body).toEqual({ + target: Target.Universal, + url: "https://example.com", + headless: "png", + }); + }); + it("does not override explicit parse or markdown", () => { const body: Record = { target: Target.GoogleSearch, diff --git a/tests/platform/errors/cli-usage-error.test.ts b/tests/platform/errors/cli-usage-error.test.ts new file mode 100644 index 0000000..39847a4 --- /dev/null +++ b/tests/platform/errors/cli-usage-error.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { CliUsageError } from "../../../src/platform/errors/cli-usage-error.js"; + +describe("CliUsageError", () => { + it("sets name and message", () => { + const err = new CliUsageError("bad flag"); + + expect(err).toBeInstanceOf(Error); + expect(err.name).toBe("CliUsageError"); + expect(err.message).toBe("bad flag"); + }); +}); diff --git a/tests/platform/services/handle-cli-error.test.ts b/tests/platform/services/handle-cli-error.test.ts index e356a99..f83f7e3 100644 --- a/tests/platform/services/handle-cli-error.test.ts +++ b/tests/platform/services/handle-cli-error.test.ts @@ -7,10 +7,8 @@ import { } from "@decodo/sdk-ts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { AuthRequiredError } from "../../../src/auth/errors/auth-required-error.js"; -import { - CliUsageError, - handleCliError, -} from "../../../src/platform/services/handle-cli-error.js"; +import { CliUsageError } from "../../../src/platform/errors/cli-usage-error.js"; +import { handleCliError } from "../../../src/platform/services/handle-cli-error.js"; describe("handleCliError", () => { let exitCode: number | undefined; @@ -90,6 +88,22 @@ describe("handleCliError", () => { expect(exitCode).toBe(7); }); + it("maps syscall-coded network failures in the cause chain to exit code 7", () => { + const cause = Object.assign( + new Error("getaddrinfo ENOTFOUND api.decodo.com"), + { + code: "ENOTFOUND", + } + ); + const err = Object.assign(new TypeError("fetch failed"), { cause }); + + expect(() => handleCliError(err)).toThrow("process.exit:7"); + + expect(exitCode).toBe(7); + expect(stderr.join("\n")).toContain("fetch failed"); + expect(stderr.join("\n")).toContain("ENOTFOUND"); + }); + it("maps explicit usage errors to exit code 2", () => { expect(() => handleCliError(new CliUsageError("bad flag"))).toThrow( "process.exit:2" diff --git a/tests/platform/services/prompt-hidden.test.ts b/tests/platform/services/prompt-hidden.test.ts index d400f5e..4c6ebc1 100644 --- a/tests/platform/services/prompt-hidden.test.ts +++ b/tests/platform/services/prompt-hidden.test.ts @@ -2,15 +2,27 @@ import { stdin } from "node:process"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { promptHidden } from "../../../src/platform/services/prompt-hidden.js"; +const { questionMock, closeListeners } = vi.hoisted(() => ({ + questionMock: vi.fn(), + closeListeners: [] as Array<() => void>, +})); + vi.mock("node:readline/promises", () => ({ createInterface: vi.fn(() => ({ - question: vi.fn().mockResolvedValue(" piped-token "), + question: questionMock, close: vi.fn(), + once: vi.fn((event: string, listener: () => void) => { + if (event === "close") { + closeListeners.push(listener); + } + }), })), })); describe("promptHidden", () => { beforeEach(() => { + closeListeners.length = 0; + questionMock.mockReset(); Object.defineProperty(stdin, "isTTY", { configurable: true, value: false }); }); @@ -19,6 +31,19 @@ describe("promptHidden", () => { }); it("falls back to readline when stdin is not a TTY", async () => { + questionMock.mockResolvedValue(" piped-token "); + await expect(promptHidden("Token: ")).resolves.toBe("piped-token"); }); + + it("rejects as a usage error when stdin closes with no input (EOF)", async () => { + questionMock.mockReturnValue(new Promise(() => undefined)); + + const pending = promptHidden("Token: "); + for (const listener of closeListeners) { + listener(); + } + + await expect(pending).rejects.toThrow("No auth token provided on stdin."); + }); }); diff --git a/tests/scrape/services/command-builder.test.ts b/tests/scrape/services/command-builder.test.ts index be9dca6..3108a7c 100644 --- a/tests/scrape/services/command-builder.test.ts +++ b/tests/scrape/services/command-builder.test.ts @@ -9,6 +9,8 @@ import { snakeToCamel } from "../../../src/scrape/services/naming.js"; const schema = BundledSchema.shared; +const INVALID_JSON_FLAG_ERROR = /--headers expects valid JSON/; + describe("configureTargetCommand", () => { it("adds a required input argument for google_search", () => { const command = new Command("google-search"); @@ -87,6 +89,46 @@ describe("buildScrapeBody", () => { }); }); + it("registers object/array flags with a JSON-parsing argParser", () => { + const command = new Command("universal"); + command.exitOverride(); + configureTargetCommand(command, "universal", schema); + command.action(() => undefined); + + const headersOption = command.options.find( + (opt) => opt.long === "--headers" + ); + expect(headersOption?.flags).toContain(""); + + command.parse( + [ + "https://example.com", + "--headers", + '{"X-Test":"1"}', + "--successful-status-codes", + "[200,204]", + ], + { from: "user" } + ); + + const opts = command.opts(); + expect(opts.headers).toEqual({ "X-Test": "1" }); + expect(opts.successfulStatusCodes).toEqual([200, 204]); + }); + + it("rejects invalid JSON for object/array flags as a usage error", () => { + const command = new Command("universal"); + command.exitOverride(); + configureTargetCommand(command, "universal", schema); + command.action(() => undefined); + + expect(() => + command.parse(["https://example.com", "--headers", "not-json"], { + from: "user", + }) + ).toThrow(INVALID_JSON_FLAG_ERROR); + }); + it("does not override explicit parse: false from options", () => { const config = configureTargetCommand( new Command("google-search"), diff --git a/tests/scrape/services/run-target-scrape.test.ts b/tests/scrape/services/run-target-scrape.test.ts index a45df9f..360b960 100644 --- a/tests/scrape/services/run-target-scrape.test.ts +++ b/tests/scrape/services/run-target-scrape.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ConfigParseError } from "../../../src/auth/errors/config-parse-error.js"; import { resolveAuthToken } from "../../../src/auth/services/resolve-token.js"; import { attachScrapeOutputOptions } from "../../../src/output/commands/attach-output-options.js"; +import { writeBinaryOutput } from "../../../src/platform/services/write-binary.js"; import { createDecodoClient } from "../../../src/scrape/services/client.js"; import { createTargetAction } from "../../../src/scrape/services/run-target-scrape.js"; @@ -15,6 +16,14 @@ vi.mock("../../../src/scrape/services/client.js", () => ({ createDecodoClient: vi.fn(), })); +vi.mock("../../../src/platform/services/write-binary.js", () => ({ + writeBinaryOutput: vi.fn(), +})); + +const PNG_BASE64 = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]).toString("base64"); + const RESPONSE_LATENCY_LOG_PATTERN = /\[verbose\] response latency_ms=\d+\n/; describe("createTargetAction", () => { @@ -190,6 +199,42 @@ describe("createTargetAction", () => { expect(exitCode).toBe(4); }); + it("writes PNG output when the built body requests headless png", async () => { + const scrape = vi.fn().mockResolvedValue({ + results: [{ content: PNG_BASE64 }], + }); + vi.mocked(createDecodoClient).mockReturnValue({ + webScrapingApi: { scrape }, + } as never); + + const universal = new Command("universal") + .argument("") + .option("--headless ") + .action(createTargetAction("universal", BundledSchema.shared)); + attachScrapeOutputOptions(universal); + + const program = new Command() + .option("--token ") + .addCommand(universal); + + await program.parseAsync( + [ + "universal", + "https://example.com", + "--headless", + "png", + "--token", + "test-token", + ], + { from: "user" } + ); + + expect(writeBinaryOutput).toHaveBeenCalledTimes(1); + const [buffer] = vi.mocked(writeBinaryOutput).mock.calls[0] ?? []; + expect(Buffer.isBuffer(buffer)).toBe(true); + expect(stdout).toBeUndefined(); + }); + it("handles targets without a primary input argument", async () => { const scrape = vi.fn().mockResolvedValue({ results: [{ content: { ok: true } }],