Replace shell-quote with shlex for proper $'...' ANSI-C quoting support

shell-quote doesn't support $'...' (ANSI-C quoting), which is a bash
extension used by browsers when copying requests as cURL. It misinterprets
the $ as variable expansion, mangling JSON bodies and escape sequences
like \n, \uXXXX, \r, etc.

Switches to shlex which handles $'...' natively. Adds splitCommands()
to handle command separation (;, newlines) that shell-quote used to do,
and a tokenize() wrapper with fallback for malformed input.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Gregory Schier
2026-02-09 07:02:22 -08:00
parent 66942eaf2c
commit 9b25079f10
4 changed files with 164 additions and 100 deletions

23
package-lock.json generated
View File

@@ -3922,13 +3922,6 @@
"@types/react": "*" "@types/react": "*"
} }
}, },
"node_modules/@types/shell-quote": {
"version": "1.7.5",
"resolved": "https://registry.npmjs.org/@types/shell-quote/-/shell-quote-1.7.5.tgz",
"integrity": "sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/unist": { "node_modules/@types/unist": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@@ -13405,6 +13398,7 @@
"version": "1.8.3", "version": "1.8.3",
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
"integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==",
"dev": true,
"license": "MIT", "license": "MIT",
"engines": { "engines": {
"node": ">= 0.4" "node": ">= 0.4"
@@ -13413,6 +13407,12 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/shlex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shlex/-/shlex-3.0.0.tgz",
"integrity": "sha512-jHPXQQk9d/QXCvJuLPYMOYWez3c43sORAgcIEoV7bFv5AJSJRAOyw5lQO12PMfd385qiLRCaDt7OtEzgrIGZUA==",
"license": "MIT"
},
"node_modules/should": { "node_modules/should": {
"version": "13.2.3", "version": "13.2.3",
"resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz", "resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz",
@@ -15955,7 +15955,7 @@
}, },
"plugins-external/httpsnippet": { "plugins-external/httpsnippet": {
"name": "@yaak/httpsnippet", "name": "@yaak/httpsnippet",
"version": "1.0.0", "version": "1.0.3",
"dependencies": { "dependencies": {
"@readme/httpsnippet": "^11.0.0" "@readme/httpsnippet": "^11.0.0"
}, },
@@ -15983,7 +15983,7 @@
}, },
"plugins-external/mcp-server": { "plugins-external/mcp-server": {
"name": "@yaak/mcp-server", "name": "@yaak/mcp-server",
"version": "0.1.7", "version": "0.2.1",
"dependencies": { "dependencies": {
"@hono/mcp": "^0.2.3", "@hono/mcp": "^0.2.3",
"@hono/node-server": "^1.19.7", "@hono/node-server": "^1.19.7",
@@ -16080,10 +16080,7 @@
"name": "@yaak/importer-curl", "name": "@yaak/importer-curl",
"version": "0.1.0", "version": "0.1.0",
"dependencies": { "dependencies": {
"shell-quote": "^1.8.1" "shlex": "^3.0.0"
},
"devDependencies": {
"@types/shell-quote": "^1.7.5"
} }
}, },
"plugins/importer-insomnia": { "plugins/importer-insomnia": {

View File

@@ -10,9 +10,6 @@
"test": "vitest --run tests" "test": "vitest --run tests"
}, },
"dependencies": { "dependencies": {
"shell-quote": "^1.8.1" "shlex": "^3.0.0"
},
"devDependencies": {
"@types/shell-quote": "^1.7.5"
} }
} }

View File

@@ -7,8 +7,7 @@ import type {
PluginDefinition, PluginDefinition,
Workspace, Workspace,
} from '@yaakapp/api'; } from '@yaakapp/api';
import type { ControlOperator, ParseEntry } from 'shell-quote'; import { split } from 'shlex';
import { parse } from 'shell-quote';
type AtLeast<T, K extends keyof T> = Partial<T> & Pick<T, K>; type AtLeast<T, K extends keyof T> = Partial<T> & Pick<T, K>;
@@ -56,31 +55,101 @@ export const plugin: PluginDefinition = {
}; };
/** /**
* Decodes escape sequences in shell $'...' strings * Splits raw input into individual shell command strings.
* Handles Unicode escape sequences (\uXXXX) and common escape codes * Handles line continuations, semicolons, and newline-separated curl commands.
*/ */
function decodeShellString(str: string): string { function splitCommands(rawData: string): string[] {
return str // Join line continuations (backslash-newline, and backslash-CRLF for Windows)
.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) const joined = rawData.replace(/\\\r?\n/g, ' ');
.replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
.replace(/\\n/g, '\n') // Split on semicolons and newlines to separate commands
.replace(/\\r/g, '\r') const commands: string[] = [];
.replace(/\\t/g, '\t') let current = '';
.replace(/\\'/g, "'") let inSingleQuote = false;
.replace(/\\"/g, '"') let inDoubleQuote = false;
.replace(/\\\\/g, '\\'); let inDollarQuote = false;
for (let i = 0; i < joined.length; i++) {
const ch = joined[i]!;
const next = joined[i + 1];
// Track quoting state to avoid splitting inside quoted strings
if (!inDoubleQuote && !inDollarQuote && ch === "'" && !inSingleQuote) {
inSingleQuote = true;
current += ch;
continue;
}
if (inSingleQuote && ch === "'") {
inSingleQuote = false;
current += ch;
continue;
}
if (!inSingleQuote && !inDollarQuote && ch === '"' && !inDoubleQuote) {
inDoubleQuote = true;
current += ch;
continue;
}
if (inDoubleQuote && ch === '"' && joined[i - 1] !== '\\') {
inDoubleQuote = false;
current += ch;
continue;
}
if (!inSingleQuote && !inDoubleQuote && !inDollarQuote && ch === '$' && next === "'") {
inDollarQuote = true;
current += ch + next;
i++; // Skip the opening quote
continue;
}
if (inDollarQuote && ch === "'" && joined[i - 1] !== '\\') {
inDollarQuote = false;
current += ch;
continue;
}
const inQuote = inSingleQuote || inDoubleQuote || inDollarQuote;
// Split on ;, newline, or CRLF when not inside quotes
if (!inQuote && (ch === ';' || ch === '\n' || (ch === '\r' && next === '\n'))) {
if (ch === '\r') i++; // Skip the \n in \r\n
if (current.trim()) {
commands.push(current.trim());
}
current = '';
continue;
}
current += ch;
}
if (current.trim()) {
commands.push(current.trim());
}
return commands;
} }
/** /**
* Checks if a string might contain escape sequences that need decoding * Tokenizes a single shell command string using shlex.
* If so, decodes them; otherwise returns the string as-is * Falls back to basic whitespace splitting if shlex fails (e.g., unmatched quotes).
*/ */
function maybeDecodeEscapeSequences(str: string): string { function tokenize(command: string): string[] {
// Check if the string contains escape sequences that shell-quote might not handle try {
if (str.includes('\\u') || str.includes('\\x')) { return split(command);
return decodeShellString(str); } catch {
// shlex is strict about unmatched quotes. Fall back to basic splitting
// and strip surrounding quotes that shell-like parsing would normally remove.
return command.split(/\s+/).filter(Boolean).map((token) => {
// Strip matching surrounding quotes
if (
(token.startsWith('"') && token.endsWith('"')) ||
(token.startsWith("'") && token.endsWith("'"))
) {
return token.slice(1, -1);
}
// Strip unmatched quotes (e.g., from malformed input)
return token.replace(/['"]/g, '');
});
} }
return str;
} }
export function convertCurl(rawData: string) { export function convertCurl(rawData: string) {
@@ -88,68 +157,17 @@ export function convertCurl(rawData: string) {
return null; return null;
} }
const commands: ParseEntry[][] = []; const commands: string[][] = splitCommands(rawData).map((cmd) => {
const tokens = tokenize(cmd);
// Replace non-escaped newlines with semicolons to make parsing easier // Break up squished arguments like `-XPOST` into `-X POST`
// NOTE: This is really slow in debug build but fast in release mode return tokens.flatMap((token) => {
const normalizedData = rawData.replace(/\ncurl/g, '; curl'); if (token.startsWith('-') && !token.startsWith('--') && token.length > 2) {
return [token.slice(0, 2), token.slice(2)];
let currentCommand: ParseEntry[] = [];
const parsed = parse(normalizedData);
// Break up `-XPOST` into `-X POST`
const normalizedParseEntries = parsed.flatMap((entry) => {
if (
typeof entry === 'string' &&
entry.startsWith('-') &&
!entry.startsWith('--') &&
entry.length > 2
) {
return [entry.slice(0, 2), entry.slice(2)];
}
return entry;
});
for (const parseEntry of normalizedParseEntries) {
if (typeof parseEntry === 'string') {
if (parseEntry.startsWith('$')) {
// Handle $'...' strings from shell-quote - decode escape sequences
currentCommand.push(decodeShellString(parseEntry.slice(1)));
} else {
// Decode escape sequences that shell-quote might not handle
currentCommand.push(maybeDecodeEscapeSequences(parseEntry));
} }
continue; return token;
} });
});
if ('comment' in parseEntry) {
continue;
}
const { op } = parseEntry as { op: 'glob'; pattern: string } | { op: ControlOperator };
// `;` separates commands
if (op === ';') {
commands.push(currentCommand);
currentCommand = [];
continue;
}
if (op?.startsWith('$')) {
// Handle the case where literal like -H $'Header: \'Some Quoted Thing\''
const str = decodeShellString(op.slice(2, op.length - 1));
currentCommand.push(str);
continue;
}
if (op === 'glob') {
currentCommand.push((parseEntry as { op: 'glob'; pattern: string }).pattern);
}
}
commands.push(currentCommand);
const workspace: ExportResources['workspaces'][0] = { const workspace: ExportResources['workspaces'][0] = {
model: 'workspace', model: 'workspace',
@@ -169,12 +187,12 @@ export function convertCurl(rawData: string) {
}; };
} }
function importCommand(parseEntries: ParseEntry[], workspaceId: string) { function importCommand(parseEntries: string[], workspaceId: string) {
// ~~~~~~~~~~~~~~~~~~~~~ // // ~~~~~~~~~~~~~~~~~~~~~ //
// Collect all the flags // // Collect all the flags //
// ~~~~~~~~~~~~~~~~~~~~~ // // ~~~~~~~~~~~~~~~~~~~~~ //
const flagsByName: FlagsByName = {}; const flagsByName: FlagsByName = {};
const singletons: ParseEntry[] = []; const singletons: string[] = [];
// Start at 1 so we can skip the ^curl part // Start at 1 so we can skip the ^curl part
for (let i = 1; i < parseEntries.length; i++) { for (let i = 1; i < parseEntries.length; i++) {

View File

@@ -112,6 +112,19 @@ describe('importer-curl', () => {
}); });
}); });
test('Imports with Windows CRLF line endings', () => {
expect(
convertCurl('curl \\\r\n -X POST \\\r\n https://yaak.app'),
).toEqual({
resources: {
workspaces: [baseWorkspace()],
httpRequests: [
baseRequest({ url: 'https://yaak.app', method: 'POST' }),
],
},
});
});
test('Imports form data', () => { test('Imports form data', () => {
expect( expect(
convertCurl('curl -X POST -F "a=aaa" -F b=bbb" -F f=@filepath https://yaak.app'), convertCurl('curl -X POST -F "a=aaa" -F b=bbb" -F f=@filepath https://yaak.app'),
@@ -476,6 +489,45 @@ describe('importer-curl', () => {
}); });
}); });
test('Imports JSON body with newlines in $quotes', () => {
expect(
convertCurl(
`curl 'https://yaak.app' -H 'Content-Type: application/json' --data-raw $'{\\n "foo": "bar",\\n "baz": "qux"\\n}' -X POST`,
),
).toEqual({
resources: {
workspaces: [baseWorkspace()],
httpRequests: [
baseRequest({
url: 'https://yaak.app',
method: 'POST',
headers: [{ name: 'Content-Type', value: 'application/json', enabled: true }],
bodyType: 'application/json',
body: { text: '{\n "foo": "bar",\n "baz": "qux"\n}' },
}),
],
},
});
});
test('Imports $quoted header with escaped single quotes', () => {
expect(
convertCurl(
`curl https://yaak.app -H $'X-Custom: it\\'s a test'`,
),
).toEqual({
resources: {
workspaces: [baseWorkspace()],
httpRequests: [
baseRequest({
url: 'https://yaak.app',
headers: [{ name: 'X-Custom', value: "it's a test", enabled: true }],
}),
],
},
});
});
test('Imports multipart form data with text-only fields from --data-raw', () => { test('Imports multipart form data with text-only fields from --data-raw', () => {
const curlCommand = `curl 'http://example.com/api' \ const curlCommand = `curl 'http://example.com/api' \
-H 'Content-Type: multipart/form-data; boundary=----FormBoundary123' \ -H 'Content-Type: multipart/form-data; boundary=----FormBoundary123' \