From 9b25079f10a16707ca5a43c49b292775efaf1f5d Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Mon, 9 Feb 2026 07:02:22 -0800 Subject: [PATCH] Replace shell-quote with shlex for proper $'...' ANSI-C quoting support shell-quote doesn't support $'...' (ANSI-C quoting), which is a bash extension used by browsers when copying requests as cURL. It misinterprets the $ as variable expansion, mangling JSON bodies and escape sequences like \n, \uXXXX, \r, etc. Switches to shlex which handles $'...' natively. Adds splitCommands() to handle command separation (;, newlines) that shell-quote used to do, and a tokenize() wrapper with fallback for malformed input. Co-Authored-By: Claude Opus 4.6 --- package-lock.json | 23 ++- plugins/importer-curl/package.json | 5 +- plugins/importer-curl/src/index.ts | 184 ++++++++++++---------- plugins/importer-curl/tests/index.test.ts | 52 ++++++ 4 files changed, 164 insertions(+), 100 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0ad5040d..8833651d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3922,13 +3922,6 @@ "@types/react": "*" } }, - "node_modules/@types/shell-quote": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@types/shell-quote/-/shell-quote-1.7.5.tgz", - "integrity": "sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -13405,6 +13398,7 @@ "version": "1.8.3", "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -13413,6 +13407,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/shlex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shlex/-/shlex-3.0.0.tgz", + "integrity": "sha512-jHPXQQk9d/QXCvJuLPYMOYWez3c43sORAgcIEoV7bFv5AJSJRAOyw5lQO12PMfd385qiLRCaDt7OtEzgrIGZUA==", + "license": "MIT" + }, "node_modules/should": { "version": "13.2.3", "resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz", @@ -15955,7 +15955,7 @@ }, "plugins-external/httpsnippet": { "name": "@yaak/httpsnippet", - "version": "1.0.0", + "version": "1.0.3", "dependencies": { "@readme/httpsnippet": "^11.0.0" }, @@ -15983,7 +15983,7 @@ }, "plugins-external/mcp-server": { "name": "@yaak/mcp-server", - "version": "0.1.7", + "version": "0.2.1", "dependencies": { "@hono/mcp": "^0.2.3", "@hono/node-server": "^1.19.7", @@ -16080,10 +16080,7 @@ "name": "@yaak/importer-curl", "version": "0.1.0", "dependencies": { - "shell-quote": "^1.8.1" - }, - "devDependencies": { - "@types/shell-quote": "^1.7.5" + "shlex": "^3.0.0" } }, "plugins/importer-insomnia": { diff --git a/plugins/importer-curl/package.json b/plugins/importer-curl/package.json index abb89b97..645204ad 100644 --- a/plugins/importer-curl/package.json +++ b/plugins/importer-curl/package.json @@ -10,9 +10,6 @@ "test": "vitest --run tests" }, "dependencies": { - "shell-quote": "^1.8.1" - }, - "devDependencies": { - "@types/shell-quote": "^1.7.5" + "shlex": "^3.0.0" } } diff --git a/plugins/importer-curl/src/index.ts b/plugins/importer-curl/src/index.ts index f8b0606e..8a46883f 100644 --- a/plugins/importer-curl/src/index.ts +++ b/plugins/importer-curl/src/index.ts @@ -7,8 +7,7 @@ import type { PluginDefinition, Workspace, } from '@yaakapp/api'; -import type { ControlOperator, ParseEntry } from 'shell-quote'; -import { parse } from 'shell-quote'; +import { split } from 'shlex'; type AtLeast = Partial & Pick; @@ -56,31 +55,101 @@ export const plugin: PluginDefinition = { }; /** - * Decodes escape sequences in shell $'...' strings - * Handles Unicode escape sequences (\uXXXX) and common escape codes + * Splits raw input into individual shell command strings. + * Handles line continuations, semicolons, and newline-separated curl commands. */ -function decodeShellString(str: string): string { - return str - .replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) - .replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) - .replace(/\\n/g, '\n') - .replace(/\\r/g, '\r') - .replace(/\\t/g, '\t') - .replace(/\\'/g, "'") - .replace(/\\"/g, '"') - .replace(/\\\\/g, '\\'); +function splitCommands(rawData: string): string[] { + // Join line continuations (backslash-newline, and backslash-CRLF for Windows) + const joined = rawData.replace(/\\\r?\n/g, ' '); + + // Split on semicolons and newlines to separate commands + const commands: string[] = []; + let current = ''; + let inSingleQuote = false; + let inDoubleQuote = false; + let inDollarQuote = false; + + for (let i = 0; i < joined.length; i++) { + const ch = joined[i]!; + const next = joined[i + 1]; + + // Track quoting state to avoid splitting inside quoted strings + if (!inDoubleQuote && !inDollarQuote && ch === "'" && !inSingleQuote) { + inSingleQuote = true; + current += ch; + continue; + } + if (inSingleQuote && ch === "'") { + inSingleQuote = false; + current += ch; + continue; + } + if (!inSingleQuote && !inDollarQuote && ch === '"' && !inDoubleQuote) { + inDoubleQuote = true; + current += ch; + continue; + } + if (inDoubleQuote && ch === '"' && joined[i - 1] !== '\\') { + inDoubleQuote = false; + current += ch; + continue; + } + if (!inSingleQuote && !inDoubleQuote && !inDollarQuote && ch === '$' && next === "'") { + inDollarQuote = true; + current += ch + next; + i++; // Skip the opening quote + continue; + } + if (inDollarQuote && ch === "'" && joined[i - 1] !== '\\') { + inDollarQuote = false; + current += ch; + continue; + } + + const inQuote = inSingleQuote || inDoubleQuote || inDollarQuote; + + // Split on ;, newline, or CRLF when not inside quotes + if (!inQuote && (ch === ';' || ch === '\n' || (ch === '\r' && next === '\n'))) { + if (ch === '\r') i++; // Skip the \n in \r\n + if (current.trim()) { + commands.push(current.trim()); + } + current = ''; + continue; + } + + current += ch; + } + + if (current.trim()) { + commands.push(current.trim()); + } + + return commands; } /** - * Checks if a string might contain escape sequences that need decoding - * If so, decodes them; otherwise returns the string as-is + * Tokenizes a single shell command string using shlex. + * Falls back to basic whitespace splitting if shlex fails (e.g., unmatched quotes). */ -function maybeDecodeEscapeSequences(str: string): string { - // Check if the string contains escape sequences that shell-quote might not handle - if (str.includes('\\u') || str.includes('\\x')) { - return decodeShellString(str); +function tokenize(command: string): string[] { + try { + return split(command); + } catch { + // shlex is strict about unmatched quotes. Fall back to basic splitting + // and strip surrounding quotes that shell-like parsing would normally remove. + return command.split(/\s+/).filter(Boolean).map((token) => { + // Strip matching surrounding quotes + if ( + (token.startsWith('"') && token.endsWith('"')) || + (token.startsWith("'") && token.endsWith("'")) + ) { + return token.slice(1, -1); + } + // Strip unmatched quotes (e.g., from malformed input) + return token.replace(/['"]/g, ''); + }); } - return str; } export function convertCurl(rawData: string) { @@ -88,68 +157,17 @@ export function convertCurl(rawData: string) { return null; } - const commands: ParseEntry[][] = []; + const commands: string[][] = splitCommands(rawData).map((cmd) => { + const tokens = tokenize(cmd); - // Replace non-escaped newlines with semicolons to make parsing easier - // NOTE: This is really slow in debug build but fast in release mode - const normalizedData = rawData.replace(/\ncurl/g, '; curl'); - - let currentCommand: ParseEntry[] = []; - - const parsed = parse(normalizedData); - - // Break up `-XPOST` into `-X POST` - const normalizedParseEntries = parsed.flatMap((entry) => { - if ( - typeof entry === 'string' && - entry.startsWith('-') && - !entry.startsWith('--') && - entry.length > 2 - ) { - return [entry.slice(0, 2), entry.slice(2)]; - } - return entry; - }); - - for (const parseEntry of normalizedParseEntries) { - if (typeof parseEntry === 'string') { - if (parseEntry.startsWith('$')) { - // Handle $'...' strings from shell-quote - decode escape sequences - currentCommand.push(decodeShellString(parseEntry.slice(1))); - } else { - // Decode escape sequences that shell-quote might not handle - currentCommand.push(maybeDecodeEscapeSequences(parseEntry)); + // Break up squished arguments like `-XPOST` into `-X POST` + return tokens.flatMap((token) => { + if (token.startsWith('-') && !token.startsWith('--') && token.length > 2) { + return [token.slice(0, 2), token.slice(2)]; } - continue; - } - - if ('comment' in parseEntry) { - continue; - } - - const { op } = parseEntry as { op: 'glob'; pattern: string } | { op: ControlOperator }; - - // `;` separates commands - if (op === ';') { - commands.push(currentCommand); - currentCommand = []; - continue; - } - - if (op?.startsWith('$')) { - // Handle the case where literal like -H $'Header: \'Some Quoted Thing\'' - const str = decodeShellString(op.slice(2, op.length - 1)); - - currentCommand.push(str); - continue; - } - - if (op === 'glob') { - currentCommand.push((parseEntry as { op: 'glob'; pattern: string }).pattern); - } - } - - commands.push(currentCommand); + return token; + }); + }); const workspace: ExportResources['workspaces'][0] = { model: 'workspace', @@ -169,12 +187,12 @@ export function convertCurl(rawData: string) { }; } -function importCommand(parseEntries: ParseEntry[], workspaceId: string) { +function importCommand(parseEntries: string[], workspaceId: string) { // ~~~~~~~~~~~~~~~~~~~~~ // // Collect all the flags // // ~~~~~~~~~~~~~~~~~~~~~ // const flagsByName: FlagsByName = {}; - const singletons: ParseEntry[] = []; + const singletons: string[] = []; // Start at 1 so we can skip the ^curl part for (let i = 1; i < parseEntries.length; i++) { diff --git a/plugins/importer-curl/tests/index.test.ts b/plugins/importer-curl/tests/index.test.ts index 6f75b8e4..cd7b1ce7 100644 --- a/plugins/importer-curl/tests/index.test.ts +++ b/plugins/importer-curl/tests/index.test.ts @@ -112,6 +112,19 @@ describe('importer-curl', () => { }); }); + test('Imports with Windows CRLF line endings', () => { + expect( + convertCurl('curl \\\r\n -X POST \\\r\n https://yaak.app'), + ).toEqual({ + resources: { + workspaces: [baseWorkspace()], + httpRequests: [ + baseRequest({ url: 'https://yaak.app', method: 'POST' }), + ], + }, + }); + }); + test('Imports form data', () => { expect( convertCurl('curl -X POST -F "a=aaa" -F b=bbb" -F f=@filepath https://yaak.app'), @@ -476,6 +489,45 @@ describe('importer-curl', () => { }); }); + test('Imports JSON body with newlines in $quotes', () => { + expect( + convertCurl( + `curl 'https://yaak.app' -H 'Content-Type: application/json' --data-raw $'{\\n "foo": "bar",\\n "baz": "qux"\\n}' -X POST`, + ), + ).toEqual({ + resources: { + workspaces: [baseWorkspace()], + httpRequests: [ + baseRequest({ + url: 'https://yaak.app', + method: 'POST', + headers: [{ name: 'Content-Type', value: 'application/json', enabled: true }], + bodyType: 'application/json', + body: { text: '{\n "foo": "bar",\n "baz": "qux"\n}' }, + }), + ], + }, + }); + }); + + test('Imports $quoted header with escaped single quotes', () => { + expect( + convertCurl( + `curl https://yaak.app -H $'X-Custom: it\\'s a test'`, + ), + ).toEqual({ + resources: { + workspaces: [baseWorkspace()], + httpRequests: [ + baseRequest({ + url: 'https://yaak.app', + headers: [{ name: 'X-Custom', value: "it's a test", enabled: true }], + }), + ], + }, + }); + }); + test('Imports multipart form data with text-only fields from --data-raw', () => { const curlCommand = `curl 'http://example.com/api' \ -H 'Content-Type: multipart/form-data; boundary=----FormBoundary123' \