From ac43bdbb175e8079d6df070ed76ae1aa8be6cf1a Mon Sep 17 00:00:00 2001 From: Jen Basch Date: Tue, 18 Aug 2026 09:44:02 -0400 Subject: [PATCH] Add hex/binary/octal support to `String.toInt()` (#1808) --- .../org/pkl/core/ast/builder/AstBuilder.java | 36 ++++----- .../java/org/pkl/core/runtime/VmUtils.java | 21 ++++++ .../org/pkl/core/stdlib/base/StringNodes.java | 75 +++++++++++++------ .../LanguageSnippetTests/input/api/string.pkl | 69 ++++++++++++++++- .../output/api/reflectedDeclaration.pcf | 40 ++++++++-- .../output/api/string.pcf | 67 ++++++++++++++++- .../src/main/java/org/pkl/parser/Lexer.java | 20 ++--- stdlib/base.pkl | 20 ++++- 8 files changed, 279 insertions(+), 69 deletions(-) diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index 22836aaa9..9b2091ecd 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -572,19 +572,21 @@ public class AstBuilder extends AbstractAstBuilder { } } - private T parseNumber(IntLiteralExpr expr, BiFunction parser) { - var text = remove_(expr.getNumber()); - + private T parseInt(IntLiteralExpr expr, BiFunction parser) { + var text = + VmUtils.removeUnderscoresFromNumber(expr.getNumber(), false).toLowerCase(Locale.ROOT); var radix = 10; - if (text.startsWith("0x") || text.startsWith("0b") || text.startsWith("0o")) { + if (text.length() >= 2 && text.charAt(0) == '0') { radix = switch (text.charAt(1)) { - case 'x' -> 16; - case 'b' -> 2; - default -> 8; + case 'x', 'X' -> 16; + case 'b', 'B' -> 2; + case 'o', 'O' -> 8; + default -> 10; }; - - text = text.substring(2); + if (radix != 10) { + text = text.substring(2); + } } // relies on grammar rule nesting depth, but a breakage won't go unnoticed by tests @@ -600,7 +602,7 @@ public class AstBuilder extends AbstractAstBuilder { public IntLiteralNode visitIntLiteralExpr(IntLiteralExpr expr) { var section = createSourceSection(expr); try { - var num = parseNumber(expr, Long::parseLong); + var num = parseInt(expr, Long::parseLong); return new IntLiteralNode(section, num); } catch (NumberFormatException e) { var text = expr.getNumber(); @@ -611,7 +613,7 @@ public class AstBuilder extends AbstractAstBuilder { @Override public FloatLiteralNode visitFloatLiteralExpr(FloatLiteralExpr expr) { var section = createSourceSection(expr); - var text = remove_(expr.getNumber()); + var text = VmUtils.removeUnderscoresFromNumber(expr.getNumber(), true); // relies on grammar rule nesting depth, but a breakage won't go unnoticed by tests if (expr.parent() instanceof UnaryMinusExpr) { // handle negation here for consistency with visitIntegerLiteral @@ -627,16 +629,6 @@ public class AstBuilder extends AbstractAstBuilder { } } - private static String remove_(String number) { - var builder = new StringBuilder(number.length()); - for (var i = 0; i < number.length(); i++) { - var ch = number.charAt(i); - if (ch == '_') continue; - builder.append(ch); - } - return builder.toString(); - } - @Override public ExpressionNode visitThrowExpr(ThrowExpr expr) { return ThrowNodeGen.create(createSourceSection(expr), visitExpr(expr.getExpr())); @@ -1314,7 +1306,7 @@ public class AstBuilder extends AbstractAstBuilder { var expr = args.get(i); if (expr instanceof IntLiteralExpr intLiteralExpr && isAllByteLiterals) { try { - var byt = parseNumber(intLiteralExpr, Byte::parseByte); + var byt = parseInt(intLiteralExpr, Byte::parseByte); expressionNodes[i] = new ByteConstantValueNode(byt); } catch (NumberFormatException e) { // proceed with initializing a constant value node; we'll throw an error inside diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java index 7ddcbf5dd..0466087fb 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java @@ -1092,4 +1092,25 @@ public final class VmUtils { var truffleStackTraceElements = TruffleStackTrace.getStackTrace(e); return truffleStackTraceElements != null && truffleStackTraceElements.size() < 100; } + + /** Removes `_` from numbers to be parsed. Returns the string unmodified if it's invalid. */ + public static String removeUnderscoresFromNumber(String number, boolean allowExponents) { + if (number.indexOf('_') < 0) return number; + + var builder = new StringBuilder(); + var numberStart = true; + for (var i = 0; i < number.length(); i++) { + var c = number.charAt(i); + if (c != '_') { + builder.append(c); + } else if (numberStart) { + // invalid: _ at start or after [.eE] + return number; + } + + numberStart = c == '.' || (allowExponents && (c == 'e' || c == 'E')); + } + + return builder.toString(); + } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java index c21cb1387..c92da1ffe 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java @@ -33,6 +33,9 @@ import org.pkl.core.util.GlobResolver; import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; import org.pkl.core.util.Pair; import org.pkl.core.util.StringUtils; +import org.pkl.parser.Lexer; +import org.pkl.parser.ParserError; +import org.pkl.parser.Token; @SuppressWarnings("unused") public final class StringNodes { @@ -830,12 +833,57 @@ public final class StringNodes { } } + /** Use the lexer to parse integer values in the same forms that Pkl itself accepts */ + private static long toInt(String self) throws NumberFormatException { + try { + var lexer = new Lexer(self); + var tk = lexer.next(); + var prefix = ""; + switch (tk) { + case MINUS -> { + prefix = "-"; + tk = lexer.next(); + } + case PLUS -> tk = lexer.next(); + } + + int radix; + var text = lexer.text(); + switch (tk) { + case INT -> radix = 10; + case BIN -> { + radix = 2; + text = text.substring(2); // strip "0b" + } + case OCT -> { + radix = 8; + text = text.substring(2); // strip "0o" + } + case HEX -> { + radix = 16; + text = text.substring(2); // strip "0x" + } + default -> throw new NumberFormatException(); + } + var parsed = Long.parseLong(prefix + VmUtils.removeUnderscoresFromNumber(text, false), radix); + + // ensure no trailing garbage + if (lexer.next() != Token.EOF) { + throw new NumberFormatException(); + } + + return parsed; + } catch (ParserError ignored) { + throw new NumberFormatException(); + } + } + public abstract static class toInt extends ExternalMethod0Node { @TruffleBoundary @Specialization protected long eval(String self) { try { - return Long.parseLong(removeUnderlinesFromNumber(self)); + return toInt(self); } catch (NumberFormatException e) { throw exceptionBuilder() .evalError("cannotParseStringAs", "Int") @@ -850,7 +898,7 @@ public final class StringNodes { @Specialization protected Object eval(String self) { try { - return Long.parseLong(removeUnderlinesFromNumber(self)); + return toInt(self); } catch (NumberFormatException e) { return VmNull.withoutDefault(); } @@ -862,7 +910,7 @@ public final class StringNodes { @Specialization protected double eval(String self) { try { - return Double.parseDouble(removeUnderlinesFromNumber(self)); + return Double.parseDouble(VmUtils.removeUnderscoresFromNumber(self, true)); } catch (NumberFormatException e) { throw exceptionBuilder() .evalError("cannotParseStringAs", "Float") @@ -877,7 +925,7 @@ public final class StringNodes { @Specialization protected Object eval(String self) { try { - return Double.parseDouble(removeUnderlinesFromNumber(self)); + return Double.parseDouble(VmUtils.removeUnderscoresFromNumber(self, true)); } catch (NumberFormatException e) { return VmNull.withoutDefault(); } @@ -1031,23 +1079,4 @@ public final class StringNodes { var replacement = applyNode.executeString(mapper, regexMatch); return Matcher.quoteReplacement(replacement); } - - /** - * Removes `_` from numbers to be parsed to be compatible with how Pkl parses numbers. Will return - * the string unmodified if it's invalid. - */ - private static String removeUnderlinesFromNumber(String number) { - var builder = new StringBuilder(); - var numberStart = true; - for (var i = 0; i < number.length(); i++) { - var c = number.charAt(i); - if (c != '_') { - builder.append(c); - } else if (numberStart) return number; - - numberStart = c == '.' || c == 'e' || c == 'E'; - } - - return builder.toString(); - } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl index 6c1228694..29939fac7 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl @@ -263,18 +263,80 @@ examples { str1.codePoints } - ["toInt()"] { + ["toInt() decimal"] { "123".toInt() "-123".toInt() + "- 123".toInt() + "+123".toInt() + "+ 123".toInt() "1_2__3___".toInt() "-1_2__3___".toInt() + "+1_2__3___".toInt() "0".toInt() "-0".toInt() + "+0".toInt() + "-9223372036854775808".toInt() // math.minInt + "9223372036854775807".toInt() // math.maxInt + } + + ["toInt() hex"] { + "0x123".toInt() + "0X123".toInt() + "-0x123".toInt() + "- 0x123".toInt() + "+0x123".toInt() + "+ 0x123".toInt() + "0x1_2__3___".toInt() + "-0x1_2__3___".toInt() + "+0x1_2__3___".toInt() + "0x0".toInt() + "-0x0".toInt() + "+0x0".toInt() + "0x1E_2".toInt() // test underscore removal after 'E' succeeds for toInt (fails for toFloat) + } + + ["toInt() binary"] { + "0b101".toInt() + "0B101".toInt() + "-0b101".toInt() + "- 0b101".toInt() + "+0b101".toInt() + "+ 0b101".toInt() + "0b1_0__1___".toInt() + "-0b1_0__1___".toInt() + "+0b1_0__1___".toInt() + "0b0".toInt() + "-0b0".toInt() + "+0b0".toInt() + } + + ["toInt() octal"] { + "0o123".toInt() + "0O123".toInt() + "-0o123".toInt() + "- 0o123".toInt() + "+0o123".toInt() + "+ 0o123".toInt() + "0o1_2__3___".toInt() + "-0o1_2__3___".toInt() + "+0o1_2__3___".toInt() + "0o0".toInt() + "-0o0".toInt() + "+0o0".toInt() + } + + ["toInt() error cases"] { module.catch(() -> "1.2".toInt()) module.catch(() -> "9223372036854775808".toInt()) module.catch(() -> "-9223372036854775809".toInt()) module.catch(() -> "abc".toInt()) module.catch(() -> "_1_000".toInt()) + module.catch(() -> "0p0".toInt()) + module.catch(() -> "0x-0".toInt()) + module.catch(() -> "0x+0".toInt()) + module.catch(() -> "0x0 + 0x1".toInt()) + module.catch(() -> "".toInt()) + module.catch(() -> "0o123e4".toInt()) } ["toIntOrNull()"] { @@ -289,6 +351,11 @@ examples { "-9223372036854775809".toIntOrNull() "abc".toIntOrNull() "_1_2__3___".toIntOrNull() + "0p0".toIntOrNull() + "0x-0".toIntOrNull() + "0x+0".toIntOrNull() + "0x0 + 0x1".toIntOrNull() + "".toIntOrNull() } ["toFloat()"] { diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf index 176422757..73b2212ff 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf @@ -2255,9 +2255,15 @@ alias { displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" } docComment = """ - Parses this string as a signed decimal (base 10) integer. + Parses this string as a signed integer. - Throws if this string cannot be parsed as a signed decimal integer, + Supports integer formats supported by Pkl: + * Decimal (base 10) + * Hexadecimal (base 16) with prefix `0x` + * Binary (base 2) with prefix `0b` + * Octal (base 8) with prefix `0o` + + Throws if this string cannot be parsed, or if the integer is too large to fit into [Int]. """ annotations = List() @@ -2272,9 +2278,15 @@ alias { displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" } docComment = """ - Parses this string as a signed decimal (base 10) integer. + Parses this string as a signed integer. - Returns [null] if this string cannot be parsed as a signed decimal integer, + Supports integer formats supported by Pkl: + * Decimal (base 10) + * Hexadecimal (base 16) with prefix `0x` + * Binary (base 2) with prefix `0b` + * Octal (base 8) with prefix `0o` + + Returns [null] if this string cannot be parsed, or if the integer is too large to fit into [Int]. """ annotations = List() @@ -3192,9 +3204,15 @@ alias { displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" } docComment = """ - Parses this string as a signed decimal (base 10) integer. + Parses this string as a signed integer. - Throws if this string cannot be parsed as a signed decimal integer, + Supports integer formats supported by Pkl: + * Decimal (base 10) + * Hexadecimal (base 16) with prefix `0x` + * Binary (base 2) with prefix `0b` + * Octal (base 8) with prefix `0o` + + Throws if this string cannot be parsed, or if the integer is too large to fit into [Int]. """ annotations = List() @@ -3209,9 +3227,15 @@ alias { displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX" } docComment = """ - Parses this string as a signed decimal (base 10) integer. + Parses this string as a signed integer. - Returns [null] if this string cannot be parsed as a signed decimal integer, + Supports integer formats supported by Pkl: + * Decimal (base 10) + * Hexadecimal (base 16) with prefix `0x` + * Binary (base 2) with prefix `0b` + * Octal (base 8) with prefix `0o` + + Returns [null] if this string cannot be parsed, or if the integer is too large to fit into [Int]. """ annotations = List() diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf index deb0a782b..1c39c1683 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf @@ -213,18 +213,76 @@ examples { List() List(97, 98, 99, 100, 101, 102, 103) } - ["toInt()"] { + ["toInt() decimal"] { + 123 + -123 + -123 + 123 + 123 123 -123 123 - -123 0 0 + 0 + -9223372036854775808 + 9223372036854775807 + } + ["toInt() hex"] { + 291 + 291 + -291 + -291 + 291 + 291 + 291 + -291 + 291 + 0 + 0 + 0 + 482 + } + ["toInt() binary"] { + 5 + 5 + -5 + -5 + 5 + 5 + 5 + -5 + 5 + 0 + 0 + 0 + } + ["toInt() octal"] { + 83 + 83 + -83 + -83 + 83 + 83 + 83 + -83 + 83 + 0 + 0 + 0 + } + ["toInt() error cases"] { "Cannot parse string as `Int`. String: \"1.2\"" "Cannot parse string as `Int`. String: \"9223372036854775808\"" "Cannot parse string as `Int`. String: \"-9223372036854775809\"" "Cannot parse string as `Int`. String: \"abc\"" "Cannot parse string as `Int`. String: \"_1_000\"" + "Cannot parse string as `Int`. String: \"0p0\"" + "Cannot parse string as `Int`. String: \"0x-0\"" + "Cannot parse string as `Int`. String: \"0x+0\"" + "Cannot parse string as `Int`. String: \"0x0 + 0x1\"" + "Cannot parse string as `Int`. String: \"\"" + "Cannot parse string as `Int`. String: \"0o123e4\"" } ["toIntOrNull()"] { 123 @@ -238,6 +296,11 @@ examples { null null null + null + null + null + null + null } ["toFloat()"] { 0.0 diff --git a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java index b67c3094c..3c348a714 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java @@ -236,7 +236,7 @@ public final class Lexer { } else { throw unexpectedChar("..", ".", "...", "...?"); } - } else if (lookahead >= 48 && lookahead <= 57) { + } else if (lookahead >= '0' && lookahead <= '9') { yield lexNumber(ch); } else { yield Token.DOT; @@ -574,7 +574,7 @@ public final class Lexer { return Token.FLOAT; } - while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') { + while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') { nextChar(); } @@ -587,7 +587,7 @@ public final class Lexer { if (lookahead == '_') { throw lexError("invalidSeparatorPosition"); } - if (lookahead < 48 || lookahead > 57) { + if (lookahead < '0' || lookahead > '9') { backup(); return Token.INT; } @@ -661,10 +661,10 @@ public final class Lexer { throw lexError("invalidSeparatorPosition"); } var ch = lookahead; - if (!(ch >= 48 && ch <= 55)) { + if (!(ch >= '0' && ch <= '7')) { throw unexpectedChar(ch, "octal number"); } - while ((ch >= 48 && ch <= 55) || ch == '_') { + while ((ch >= '0' && ch <= '7') || ch == '_') { nextChar(); ch = lookahead; } @@ -677,10 +677,10 @@ public final class Lexer { if (lookahead == '_') { throw lexError("invalidSeparatorPosition"); } - if (lookahead < 48 || lookahead > 57) { + if (lookahead < '0' || lookahead > '9') { throw unexpectedChar(lookahead, "number"); } - while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') { + while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') { nextChar(); } } @@ -689,7 +689,7 @@ public final class Lexer { if (lookahead == '_') { throw lexError("invalidSeparatorPosition"); } - while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') { + while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') { nextChar(); } if (lookahead == 'e' || lookahead == 'E') { @@ -706,7 +706,9 @@ public final class Lexer { } private boolean isHex(int code) { - return (code >= 48 && code <= 57) || (code >= 97 && code <= 102) || (code >= 65 && code <= 70); + return (code >= '0' && code <= '9') + || (code >= 'a' && code <= 'f') + || (code >= 'A' && code <= 'F'); } private static boolean isIdentifierStart(int c) { diff --git a/stdlib/base.pkl b/stdlib/base.pkl index ee3e94ef2..1008ebec5 100644 --- a/stdlib/base.pkl +++ b/stdlib/base.pkl @@ -1640,15 +1640,27 @@ external class String extends Any { /// ``` external function decapitalize(): String - /// Parses this string as a signed decimal (base 10) integer. + /// Parses this string as a signed integer. /// - /// Throws if this string cannot be parsed as a signed decimal integer, + /// Supports integer formats supported by Pkl: + /// * Decimal (base 10) + /// * Hexadecimal (base 16) with prefix `0x` + /// * Binary (base 2) with prefix `0b` + /// * Octal (base 8) with prefix `0o` + /// + /// Throws if this string cannot be parsed, /// or if the integer is too large to fit into [Int]. external function toInt(): Int - /// Parses this string as a signed decimal (base 10) integer. + /// Parses this string as a signed integer. /// - /// Returns [null] if this string cannot be parsed as a signed decimal integer, + /// Supports integer formats supported by Pkl: + /// * Decimal (base 10) + /// * Hexadecimal (base 16) with prefix `0x` + /// * Binary (base 2) with prefix `0b` + /// * Octal (base 8) with prefix `0o` + /// + /// Returns [null] if this string cannot be parsed, /// or if the integer is too large to fit into [Int]. external function toIntOrNull(): Int?