Support comments in JSON body (#419)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Gregory Schier
2026-03-05 15:05:09 -08:00
committed by GitHub
parent 242f55b609
commit 267508e533
22 changed files with 954 additions and 161 deletions

View File

@@ -21,3 +21,10 @@ pub fn get_str_map<'a>(v: &'a BTreeMap<String, Value>, key: &str) -> &'a str {
Some(v) => v.as_str().unwrap_or_default(),
}
}
pub fn get_bool_map(v: &BTreeMap<String, Value>, key: &str, fallback: bool) -> bool {
match v.get(key) {
None => fallback,
Some(v) => v.as_bool().unwrap_or(fallback),
}
}

View File

@@ -29,4 +29,5 @@ tower-service = "0.3.3"
urlencoding = "2.1.3"
yaak-common = { workspace = true }
yaak-models = { workspace = true }
yaak-templates = { workspace = true }
yaak-tls = { workspace = true }

View File

@@ -9,8 +9,9 @@ use std::collections::BTreeMap;
use std::pin::Pin;
use std::time::Duration;
use tokio::io::AsyncRead;
use yaak_common::serde::{get_bool, get_str, get_str_map};
use yaak_common::serde::{get_bool, get_bool_map, get_str, get_str_map};
use yaak_models::models::HttpRequest;
use yaak_templates::strip_json_comments::{maybe_strip_json_comments, strip_json_comments};
pub(crate) const MULTIPART_BOUNDARY: &str = "------YaakFormBoundary";
@@ -134,16 +135,69 @@ pub fn append_query_params(url: &str, params: Vec<(String, String)>) -> String {
result
}
fn strip_query_params(url: &str, names: &[&str]) -> String {
// Split off fragment
let (base_and_query, fragment) = if let Some(hash_pos) = url.find('#') {
(&url[..hash_pos], Some(&url[hash_pos..]))
} else {
(url, None)
};
let result = if let Some(q_pos) = base_and_query.find('?') {
let base = &base_and_query[..q_pos];
let query = &base_and_query[q_pos + 1..];
let filtered: Vec<&str> = query
.split('&')
.filter(|pair| {
let key = pair.split('=').next().unwrap_or("");
let decoded = urlencoding::decode(key).unwrap_or_default();
!names.contains(&decoded.as_ref())
})
.collect();
if filtered.is_empty() {
base.to_string()
} else {
format!("{}?{}", base, filtered.join("&"))
}
} else {
base_and_query.to_string()
};
match fragment {
Some(f) => format!("{}{}", result, f),
None => result,
}
}
fn build_url(r: &HttpRequest) -> String {
let (url_string, params) = apply_path_placeholders(&ensure_proto(&r.url), &r.url_parameters);
append_query_params(
let mut url = append_query_params(
&url_string,
params
.iter()
.filter(|p| p.enabled && !p.name.is_empty())
.map(|p| (p.name.clone(), p.value.clone()))
.collect(),
)
);
// GraphQL GET requests encode query/variables as URL query parameters
if r.method.to_lowercase() == "get" && r.body_type.as_deref() == Some("graphql") {
url = append_graphql_query_params(&url, &r.body);
}
url
}
fn append_graphql_query_params(url: &str, body: &BTreeMap<String, serde_json::Value>) -> String {
let query = get_str_map(body, "query").to_string();
let variables = strip_json_comments(&get_str_map(body, "variables"));
let mut params = vec![("query".to_string(), query)];
if !variables.trim().is_empty() {
params.push(("variables".to_string(), variables));
}
// Strip existing query/variables params to avoid duplicates
let url = strip_query_params(url, &["query", "variables"]);
append_query_params(&url, params)
}
fn build_headers(r: &HttpRequest) -> Vec<(String, String)> {
@@ -177,7 +231,7 @@ async fn build_body(
(build_form_body(&body), Some("application/x-www-form-urlencoded".to_string()))
}
"multipart/form-data" => build_multipart_body(&body, &headers).await?,
_ if body.contains_key("text") => (build_text_body(&body), None),
_ if body.contains_key("text") => (build_text_body(&body, body_type), None),
t => {
warn!("Unsupported body type: {}", t);
(None, None)
@@ -252,13 +306,20 @@ async fn build_binary_body(
}))
}
fn build_text_body(body: &BTreeMap<String, serde_json::Value>) -> Option<SendableBodyWithMeta> {
fn build_text_body(body: &BTreeMap<String, serde_json::Value>, body_type: &str) -> Option<SendableBodyWithMeta> {
let text = get_str_map(body, "text");
if text.is_empty() {
None
} else {
Some(SendableBodyWithMeta::Bytes(Bytes::from(text.to_string())))
return None;
}
let send_comments = get_bool_map(body, "sendJsonComments", false);
let text = if !send_comments && body_type == "application/json" {
maybe_strip_json_comments(text)
} else {
text.to_string()
};
Some(SendableBodyWithMeta::Bytes(Bytes::from(text)))
}
fn build_graphql_body(
@@ -266,7 +327,7 @@ fn build_graphql_body(
body: &BTreeMap<String, serde_json::Value>,
) -> Option<SendableBodyWithMeta> {
let query = get_str_map(body, "query");
let variables = get_str_map(body, "variables");
let variables = strip_json_comments(&get_str_map(body, "variables"));
if method.to_lowercase() == "get" {
// GraphQL GET requests use query parameters, not a body
@@ -684,7 +745,7 @@ mod tests {
let mut body = BTreeMap::new();
body.insert("text".to_string(), json!("Hello, World!"));
let result = build_text_body(&body);
let result = build_text_body(&body, "application/json");
match result {
Some(SendableBodyWithMeta::Bytes(bytes)) => {
assert_eq!(bytes, Bytes::from("Hello, World!"))
@@ -698,7 +759,7 @@ mod tests {
let mut body = BTreeMap::new();
body.insert("text".to_string(), json!(""));
let result = build_text_body(&body);
let result = build_text_body(&body, "application/json");
assert!(result.is_none());
}
@@ -706,10 +767,57 @@ mod tests {
async fn test_text_body_missing() {
let body = BTreeMap::new();
let result = build_text_body(&body);
let result = build_text_body(&body, "application/json");
assert!(result.is_none());
}
#[tokio::test]
async fn test_text_body_strips_json_comments_by_default() {
let mut body = BTreeMap::new();
body.insert("text".to_string(), json!("{\n // comment\n \"foo\": \"bar\"\n}"));
let result = build_text_body(&body, "application/json");
match result {
Some(SendableBodyWithMeta::Bytes(bytes)) => {
let text = String::from_utf8_lossy(&bytes);
assert!(!text.contains("// comment"));
assert!(text.contains("\"foo\": \"bar\""));
}
_ => panic!("Expected Some(SendableBody::Bytes)"),
}
}
#[tokio::test]
async fn test_text_body_send_json_comments_when_opted_in() {
let mut body = BTreeMap::new();
body.insert("text".to_string(), json!("{\n // comment\n \"foo\": \"bar\"\n}"));
body.insert("sendJsonComments".to_string(), json!(true));
let result = build_text_body(&body, "application/json");
match result {
Some(SendableBodyWithMeta::Bytes(bytes)) => {
let text = String::from_utf8_lossy(&bytes);
assert!(text.contains("// comment"));
}
_ => panic!("Expected Some(SendableBody::Bytes)"),
}
}
#[tokio::test]
async fn test_text_body_no_strip_for_non_json() {
let mut body = BTreeMap::new();
body.insert("text".to_string(), json!("// not json\nsome text"));
let result = build_text_body(&body, "text/plain");
match result {
Some(SendableBodyWithMeta::Bytes(bytes)) => {
let text = String::from_utf8_lossy(&bytes);
assert!(text.contains("// not json"));
}
_ => panic!("Expected Some(SendableBody::Bytes)"),
}
}
#[tokio::test]
async fn test_form_urlencoded_body() -> Result<()> {
let mut body = BTreeMap::new();

View File

@@ -11,6 +11,7 @@ pub fn format_json(text: &str, tab: &str) -> String {
let mut new_json = "".to_string();
let mut depth = 0;
let mut state = FormatState::None;
let mut saw_newline_in_whitespace = false;
loop {
let rest_of_chars = chars.clone();
@@ -61,6 +62,62 @@ pub fn format_json(text: &str, tab: &str) -> String {
continue;
}
// Handle line comments (//)
if current_char == '/' && chars.peek() == Some(&'/') {
chars.next(); // Skip second /
// Collect the rest of the comment until newline
let mut comment = String::from("//");
loop {
match chars.peek() {
Some(&'\n') | None => break,
Some(_) => comment.push(chars.next().unwrap()),
}
}
// Check if the comma handler already added \n + indent
let trimmed = new_json.trim_end_matches(|c: char| c == ' ' || c == '\t');
if trimmed.ends_with(",\n") && !saw_newline_in_whitespace {
// Trailing comment on the same line as comma (e.g. "foo",// comment)
new_json.truncate(trimmed.len() - 1);
new_json.push(' ');
} else if !trimmed.ends_with('\n') && !new_json.is_empty() {
// Trailing comment after a value (no newline before us)
new_json.push(' ');
}
new_json.push_str(&comment);
new_json.push('\n');
new_json.push_str(tab.to_string().repeat(depth).as_str());
saw_newline_in_whitespace = false;
continue;
}
// Handle block comments (/* ... */)
if current_char == '/' && chars.peek() == Some(&'*') {
chars.next(); // Skip *
let mut comment = String::from("/*");
loop {
match chars.next() {
None => break,
Some('*') if chars.peek() == Some(&'/') => {
chars.next(); // Skip /
comment.push_str("*/");
break;
}
Some(c) => comment.push(c),
}
}
// If we're not already on a fresh line, add newline + indent before comment
let trimmed = new_json.trim_end_matches(|c: char| c == ' ' || c == '\t');
if !trimmed.is_empty() && !trimmed.ends_with('\n') {
new_json.push('\n');
new_json.push_str(tab.to_string().repeat(depth).as_str());
}
new_json.push_str(&comment);
// After block comment, add newline + indent for the next content
new_json.push('\n');
new_json.push_str(tab.to_string().repeat(depth).as_str());
continue;
}
match current_char {
',' => {
new_json.push(current_char);
@@ -125,20 +182,37 @@ pub fn format_json(text: &str, tab: &str) -> String {
|| current_char == '\t'
|| current_char == '\r'
{
if current_char == '\n' {
saw_newline_in_whitespace = true;
}
// Don't add these
} else {
saw_newline_in_whitespace = false;
new_json.push(current_char);
}
}
}
}
// Replace only lines containing whitespace with nothing
new_json
.lines()
.filter(|line| !line.trim().is_empty()) // Filter out whitespace-only lines
.collect::<Vec<&str>>() // Collect the non-empty lines into a vector
.join("\n") // Join the lines back into a single string
// Filter out whitespace-only lines, but preserve empty lines inside block comments
let mut result_lines: Vec<&str> = Vec::new();
let mut in_block_comment = false;
for line in new_json.lines() {
if in_block_comment {
result_lines.push(line);
if line.contains("*/") {
in_block_comment = false;
}
} else {
if line.contains("/*") && !line.contains("*/") {
in_block_comment = true;
}
if !line.trim().is_empty() {
result_lines.push(line);
}
}
}
result_lines.iter().map(|line| line.trim_end()).collect::<Vec<&str>>().join("\n")
}
#[cfg(test)]
@@ -297,6 +371,161 @@ mod tests {
r#"
{}
}
"#
.trim()
);
}
#[test]
fn test_line_comment_between_keys() {
assert_eq!(
format_json(
r#"{"foo":"bar",// a comment
"baz":"qux"}"#,
" "
),
r#"
{
"foo": "bar", // a comment
"baz": "qux"
}
"#
.trim()
);
}
#[test]
fn test_line_comment_at_end() {
assert_eq!(
format_json(
r#"{"foo":"bar" // trailing
}"#,
" "
),
r#"
{
"foo": "bar" // trailing
}
"#
.trim()
);
}
#[test]
fn test_block_comment() {
assert_eq!(
format_json(r#"{"foo":"bar",/* comment */"baz":"qux"}"#, " "),
r#"
{
"foo": "bar",
/* comment */
"baz": "qux"
}
"#
.trim()
);
}
#[test]
fn test_comment_in_array() {
assert_eq!(
format_json(
r#"[1,// item comment
2,3]"#,
" "
),
r#"
[
1, // item comment
2,
3
]
"#
.trim()
);
}
#[test]
fn test_comment_only_line() {
assert_eq!(
format_json(
r#"{
// this is a standalone comment
"foo": "bar"
}"#,
" "
),
r#"
{
// this is a standalone comment
"foo": "bar"
}
"#
.trim()
);
}
#[test]
fn test_multiline_block_comment() {
assert_eq!(
format_json(
r#"{
"foo": "bar"
/**
Hello World!
Hi there
*/
}"#,
" "
),
r#"
{
"foo": "bar"
/**
Hello World!
Hi there
*/
}
"#
.trim()
);
}
// NOTE: trailing whitespace on output lines is trimmed by the formatter.
// We can't easily add a test for this because raw string literals get
// trailing whitespace stripped by the editor/linter.
#[test]
fn test_comment_inside_string_ignored() {
assert_eq!(
format_json(r#"{"foo":"// not a comment","bar":"/* also not */"}"#, " "),
r#"
{
"foo": "// not a comment",
"bar": "/* also not */"
}
"#
.trim()
);
}
#[test]
fn test_comment_on_line_after_comma() {
assert_eq!(
format_json(
r#"{
"a": "aaa",
// "b": "bbb"
}"#,
" "
),
r#"
{
"a": "aaa",
// "b": "bbb"
}
"#
.trim()
);

View File

@@ -1,6 +1,7 @@
pub mod error;
pub mod escape;
pub mod format_json;
pub mod strip_json_comments;
pub mod parser;
pub mod renderer;
pub mod wasm;

View File

@@ -0,0 +1,318 @@
/// Strips JSON comments only if the result is valid JSON. If stripping comments
/// produces invalid JSON, the original text is returned unchanged.
pub fn maybe_strip_json_comments(text: &str) -> String {
let stripped = strip_json_comments(text);
if serde_json::from_str::<serde_json::Value>(&stripped).is_ok() {
stripped
} else {
text.to_string()
}
}
/// Strips comments from JSONC, preserving the original formatting as much as possible.
///
/// - Trailing comments on a line are removed (along with preceding whitespace)
/// - Whole-line comments are removed, including the line itself
/// - Block comments are removed, including any lines that become empty
/// - Comments inside strings and template tags are left alone
pub fn strip_json_comments(text: &str) -> String {
let mut chars = text.chars().peekable();
let mut result = String::with_capacity(text.len());
let mut in_string = false;
let mut in_template_tag = false;
loop {
let current_char = match chars.next() {
None => break,
Some(c) => c,
};
// Handle JSON strings
if in_string {
result.push(current_char);
match current_char {
'"' => in_string = false,
'\\' => {
if let Some(c) = chars.next() {
result.push(c);
}
}
_ => {}
}
continue;
}
// Handle template tags
if in_template_tag {
result.push(current_char);
if current_char == ']' && chars.peek() == Some(&'}') {
result.push(chars.next().unwrap());
in_template_tag = false;
}
continue;
}
// Check for template tag start
if current_char == '$' && chars.peek() == Some(&'{') {
let mut lookahead = chars.clone();
lookahead.next(); // skip {
if lookahead.peek() == Some(&'[') {
in_template_tag = true;
result.push(current_char);
result.push(chars.next().unwrap()); // {
result.push(chars.next().unwrap()); // [
continue;
}
}
// Check for line comment
if current_char == '/' && chars.peek() == Some(&'/') {
chars.next(); // skip second /
// Consume until newline
loop {
match chars.peek() {
Some(&'\n') | None => break,
Some(_) => {
chars.next();
}
}
}
// Trim trailing whitespace that preceded the comment
let trimmed_len = result.trim_end_matches(|c: char| c == ' ' || c == '\t').len();
result.truncate(trimmed_len);
continue;
}
// Check for block comment
if current_char == '/' && chars.peek() == Some(&'*') {
chars.next(); // skip *
// Consume until */
loop {
match chars.next() {
None => break,
Some('*') if chars.peek() == Some(&'/') => {
chars.next(); // skip /
break;
}
Some(_) => {}
}
}
// Trim trailing whitespace that preceded the comment
let trimmed_len = result.trim_end_matches(|c: char| c == ' ' || c == '\t').len();
result.truncate(trimmed_len);
// Skip whitespace/newline after the block comment if the next line is content
// (this handles the case where the block comment is on its own line)
continue;
}
if current_char == '"' {
in_string = true;
}
result.push(current_char);
}
// Remove lines that are now empty (were comment-only lines)
let result = result
.lines()
.filter(|line| !line.trim().is_empty())
.collect::<Vec<&str>>()
.join("\n");
// Remove trailing commas before } or ]
strip_trailing_commas(&result)
}
/// Removes trailing commas before closing braces/brackets, respecting strings.
fn strip_trailing_commas(text: &str) -> String {
let mut result = String::with_capacity(text.len());
let chars: Vec<char> = text.chars().collect();
let mut i = 0;
let mut in_string = false;
while i < chars.len() {
let ch = chars[i];
if in_string {
result.push(ch);
match ch {
'"' => in_string = false,
'\\' => {
i += 1;
if i < chars.len() {
result.push(chars[i]);
}
}
_ => {}
}
i += 1;
continue;
}
if ch == '"' {
in_string = true;
result.push(ch);
i += 1;
continue;
}
if ch == ',' {
// Look ahead past whitespace/newlines for } or ]
let mut j = i + 1;
while j < chars.len() && chars[j].is_whitespace() {
j += 1;
}
if j < chars.len() && (chars[j] == '}' || chars[j] == ']') {
// Skip the comma
i += 1;
continue;
}
}
result.push(ch);
i += 1;
}
result
}
#[cfg(test)]
mod tests {
use crate::strip_json_comments::strip_json_comments;
#[test]
fn test_no_comments() {
let input = r#"{
"foo": "bar",
"baz": 123
}"#;
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_trailing_line_comment() {
assert_eq!(
strip_json_comments(r#"{
"foo": "bar", // this is a comment
"baz": 123
}"#),
r#"{
"foo": "bar",
"baz": 123
}"#
);
}
#[test]
fn test_whole_line_comment() {
assert_eq!(
strip_json_comments(r#"{
// this is a comment
"foo": "bar"
}"#),
r#"{
"foo": "bar"
}"#
);
}
#[test]
fn test_inline_block_comment() {
assert_eq!(
strip_json_comments(r#"{
"foo": /* a comment */ "bar"
}"#),
r#"{
"foo": "bar"
}"#
);
}
#[test]
fn test_whole_line_block_comment() {
assert_eq!(
strip_json_comments(r#"{
/* a comment */
"foo": "bar"
}"#),
r#"{
"foo": "bar"
}"#
);
}
#[test]
fn test_multiline_block_comment() {
assert_eq!(
strip_json_comments(r#"{
/**
* Hello World!
*/
"foo": "bar"
}"#),
r#"{
"foo": "bar"
}"#
);
}
#[test]
fn test_comment_inside_string_preserved() {
let input = r#"{
"foo": "// not a comment",
"bar": "/* also not */"
}"#;
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_comment_inside_template_tag_preserved() {
let input = r#"{
"foo": ${[ fn("// hi", "/* hey */") ]}
}"#;
assert_eq!(strip_json_comments(input), input);
}
#[test]
fn test_multiple_comments() {
assert_eq!(
strip_json_comments(r#"{
// first comment
"foo": "bar", // trailing
/* block */
"baz": 123
}"#),
r#"{
"foo": "bar",
"baz": 123
}"#
);
}
#[test]
fn test_trailing_comma_after_comment_removed() {
assert_eq!(
strip_json_comments(r#"{
"a": "aaa",
// "b": "bbb"
}"#),
r#"{
"a": "aaa"
}"#
);
}
#[test]
fn test_trailing_comma_in_array() {
assert_eq!(
strip_json_comments(r#"[1, 2, /* 3 */]"#),
r#"[1, 2]"#
);
}
#[test]
fn test_comma_inside_string_preserved() {
let input = r#"{"a": "hello,}"#;
assert_eq!(strip_json_comments(input), input);
}
}

View File

@@ -2,7 +2,7 @@ use log::info;
use serde_json::Value;
use std::collections::BTreeMap;
use yaak_http::path_placeholders::apply_path_placeholders;
use yaak_models::models::{Environment, HttpRequest, HttpRequestHeader, HttpUrlParameter};
use yaak_models::models::{Environment, GrpcRequest, HttpRequest, HttpRequestHeader, HttpUrlParameter};
use yaak_models::render::make_vars_hashmap;
use yaak_templates::{RenderOptions, TemplateCallback, parse_and_render, render_json_value_raw};
@@ -89,6 +89,64 @@ pub async fn render_http_request<T: TemplateCallback>(
Ok(HttpRequest { url, url_parameters, headers, body, authentication, ..request.to_owned() })
}
pub async fn render_grpc_request<T: TemplateCallback>(
r: &GrpcRequest,
environment_chain: Vec<Environment>,
cb: &T,
opt: &RenderOptions,
) -> yaak_templates::error::Result<GrpcRequest> {
let vars = &make_vars_hashmap(environment_chain);
let mut metadata = Vec::new();
for p in r.metadata.clone() {
if !p.enabled {
continue;
}
metadata.push(HttpRequestHeader {
enabled: p.enabled,
name: parse_and_render(p.name.as_str(), vars, cb, opt).await?,
value: parse_and_render(p.value.as_str(), vars, cb, opt).await?,
id: p.id,
})
}
let authentication = {
let mut disabled = false;
let mut auth = BTreeMap::new();
match r.authentication.get("disabled") {
Some(Value::Bool(true)) => {
disabled = true;
}
Some(Value::String(tmpl)) => {
disabled = parse_and_render(tmpl.as_str(), vars, cb, opt)
.await
.unwrap_or_default()
.is_empty();
info!(
"Rendering authentication.disabled as a template: {disabled} from \"{tmpl}\""
);
}
_ => {}
}
if disabled {
auth.insert("disabled".to_string(), Value::Bool(true));
} else {
for (k, v) in r.authentication.clone() {
if k == "disabled" {
auth.insert(k, Value::Bool(false));
} else {
auth.insert(k, render_json_value_raw(v, vars, cb, opt).await?);
}
}
}
auth
};
let url = parse_and_render(r.url.as_str(), vars, cb, opt).await?;
Ok(GrpcRequest { url, metadata, authentication, ..r.to_owned() })
}
fn strip_disabled_form_entries(v: Value) -> Value {
match v {
Value::Array(items) => Value::Array(