Add hex/binary/octal support to String.toInt() (#1808)

pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java

@@ -572,20 +572,22 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
     }
   }
 
-  private <T> T parseNumber(IntLiteralExpr expr, BiFunction<String, Integer, T> parser) {
+  private <T> T parseInt(IntLiteralExpr expr, BiFunction<String, Integer, T> parser) {
-    var text = remove_(expr.getNumber());
+    var text =
-
+        VmUtils.removeUnderscoresFromNumber(expr.getNumber(), false).toLowerCase(Locale.ROOT);
     var radix = 10;
-    if (text.startsWith("0x") || text.startsWith("0b") || text.startsWith("0o")) {
+    if (text.length() >= 2 && text.charAt(0) == '0') {
       radix =
           switch (text.charAt(1)) {
-            case 'x' -> 16;
+            case 'x', 'X' -> 16;
-            case 'b' -> 2;
+            case 'b', 'B' -> 2;
-            default -> 8;
+            case 'o', 'O' -> 8;
+            default -> 10;
           };
-
+      if (radix != 10) {
         text = text.substring(2);
       }
+    }
 
     // relies on grammar rule nesting depth, but a breakage won't go unnoticed by tests
     if (expr.parent() instanceof UnaryMinusExpr) {
@@ -600,7 +602,7 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
   public IntLiteralNode visitIntLiteralExpr(IntLiteralExpr expr) {
     var section = createSourceSection(expr);
     try {
-      var num = parseNumber(expr, Long::parseLong);
+      var num = parseInt(expr, Long::parseLong);
       return new IntLiteralNode(section, num);
     } catch (NumberFormatException e) {
       var text = expr.getNumber();
@@ -611,7 +613,7 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
   @Override
   public FloatLiteralNode visitFloatLiteralExpr(FloatLiteralExpr expr) {
     var section = createSourceSection(expr);
-    var text = remove_(expr.getNumber());
+    var text = VmUtils.removeUnderscoresFromNumber(expr.getNumber(), true);
     // relies on grammar rule nesting depth, but a breakage won't go unnoticed by tests
     if (expr.parent() instanceof UnaryMinusExpr) {
       // handle negation here for consistency with visitIntegerLiteral
@@ -627,16 +629,6 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
     }
   }
 
-  private static String remove_(String number) {
-    var builder = new StringBuilder(number.length());
-    for (var i = 0; i < number.length(); i++) {
-      var ch = number.charAt(i);
-      if (ch == '_') continue;
-      builder.append(ch);
-    }
-    return builder.toString();
-  }
-
   @Override
   public ExpressionNode visitThrowExpr(ThrowExpr expr) {
     return ThrowNodeGen.create(createSourceSection(expr), visitExpr(expr.getExpr()));
@@ -1314,7 +1306,7 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
       var expr = args.get(i);
       if (expr instanceof IntLiteralExpr intLiteralExpr && isAllByteLiterals) {
         try {
-          var byt = parseNumber(intLiteralExpr, Byte::parseByte);
+          var byt = parseInt(intLiteralExpr, Byte::parseByte);
           expressionNodes[i] = new ByteConstantValueNode(byt);
         } catch (NumberFormatException e) {
           // proceed with initializing a constant value node; we'll throw an error inside

pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java

@@ -1092,4 +1092,25 @@ public final class VmUtils {
     var truffleStackTraceElements = TruffleStackTrace.getStackTrace(e);
     return truffleStackTraceElements != null && truffleStackTraceElements.size() < 100;
   }
+
+  /** Removes `_` from numbers to be parsed. Returns the string unmodified if it's invalid. */
+  public static String removeUnderscoresFromNumber(String number, boolean allowExponents) {
+    if (number.indexOf('_') < 0) return number;
+
+    var builder = new StringBuilder();
+    var numberStart = true;
+    for (var i = 0; i < number.length(); i++) {
+      var c = number.charAt(i);
+      if (c != '_') {
+        builder.append(c);
+      } else if (numberStart) {
+        // invalid: _ at start or after [.eE]
+        return number;
+      }
+
+      numberStart = c == '.' || (allowExponents && (c == 'e' || c == 'E'));
+    }
+
+    return builder.toString();
+  }
 }

pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java

@@ -33,6 +33,9 @@ import org.pkl.core.util.GlobResolver;
 import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
 import org.pkl.core.util.Pair;
 import org.pkl.core.util.StringUtils;
+import org.pkl.parser.Lexer;
+import org.pkl.parser.ParserError;
+import org.pkl.parser.Token;
 
 @SuppressWarnings("unused")
 public final class StringNodes {
@@ -830,12 +833,57 @@ public final class StringNodes {
     }
   }
 
+  /** Use the lexer to parse integer values in the same forms that Pkl itself accepts */
+  private static long toInt(String self) throws NumberFormatException {
+    try {
+      var lexer = new Lexer(self);
+      var tk = lexer.next();
+      var prefix = "";
+      switch (tk) {
+        case MINUS -> {
+          prefix = "-";
+          tk = lexer.next();
+        }
+        case PLUS -> tk = lexer.next();
+      }
+
+      int radix;
+      var text = lexer.text();
+      switch (tk) {
+        case INT -> radix = 10;
+        case BIN -> {
+          radix = 2;
+          text = text.substring(2); // strip "0b"
+        }
+        case OCT -> {
+          radix = 8;
+          text = text.substring(2); // strip "0o"
+        }
+        case HEX -> {
+          radix = 16;
+          text = text.substring(2); // strip "0x"
+        }
+        default -> throw new NumberFormatException();
+      }
+      var parsed = Long.parseLong(prefix + VmUtils.removeUnderscoresFromNumber(text, false), radix);
+
+      // ensure no trailing garbage
+      if (lexer.next() != Token.EOF) {
+        throw new NumberFormatException();
+      }
+
+      return parsed;
+    } catch (ParserError ignored) {
+      throw new NumberFormatException();
+    }
+  }
+
   public abstract static class toInt extends ExternalMethod0Node {
     @TruffleBoundary
     @Specialization
     protected long eval(String self) {
       try {
-        return Long.parseLong(removeUnderlinesFromNumber(self));
+        return toInt(self);
       } catch (NumberFormatException e) {
         throw exceptionBuilder()
             .evalError("cannotParseStringAs", "Int")
@@ -850,7 +898,7 @@ public final class StringNodes {
     @Specialization
     protected Object eval(String self) {
       try {
-        return Long.parseLong(removeUnderlinesFromNumber(self));
+        return toInt(self);
       } catch (NumberFormatException e) {
         return VmNull.withoutDefault();
       }
@@ -862,7 +910,7 @@ public final class StringNodes {
     @Specialization
     protected double eval(String self) {
       try {
-        return Double.parseDouble(removeUnderlinesFromNumber(self));
+        return Double.parseDouble(VmUtils.removeUnderscoresFromNumber(self, true));
       } catch (NumberFormatException e) {
         throw exceptionBuilder()
             .evalError("cannotParseStringAs", "Float")
@@ -877,7 +925,7 @@ public final class StringNodes {
     @Specialization
     protected Object eval(String self) {
       try {
-        return Double.parseDouble(removeUnderlinesFromNumber(self));
+        return Double.parseDouble(VmUtils.removeUnderscoresFromNumber(self, true));
       } catch (NumberFormatException e) {
         return VmNull.withoutDefault();
       }
@@ -1031,23 +1079,4 @@ public final class StringNodes {
     var replacement = applyNode.executeString(mapper, regexMatch);
     return Matcher.quoteReplacement(replacement);
   }
-
-  /**
-   * Removes `_` from numbers to be parsed to be compatible with how Pkl parses numbers. Will return
-   * the string unmodified if it's invalid.
-   */
-  private static String removeUnderlinesFromNumber(String number) {
-    var builder = new StringBuilder();
-    var numberStart = true;
-    for (var i = 0; i < number.length(); i++) {
-      var c = number.charAt(i);
-      if (c != '_') {
-        builder.append(c);
-      } else if (numberStart) return number;
-
-      numberStart = c == '.' || c == 'e' || c == 'E';
-    }
-
-    return builder.toString();
-  }
 }

pkl-core/src/test/files/LanguageSnippetTests/input/api/string.pkl

@@ -263,18 +263,80 @@ examples {
     str1.codePoints
   }
 
-  ["toInt()"] {
+  ["toInt() decimal"] {
     "123".toInt()
     "-123".toInt()
+    "- 123".toInt()
+    "+123".toInt()
+    "+ 123".toInt()
     "1_2__3___".toInt()
     "-1_2__3___".toInt()
+    "+1_2__3___".toInt()
     "0".toInt()
     "-0".toInt()
+    "+0".toInt()
+    "-9223372036854775808".toInt() // math.minInt
+    "9223372036854775807".toInt() // math.maxInt
+  }
+  
+  ["toInt() hex"] {
+    "0x123".toInt()
+    "0X123".toInt()
+    "-0x123".toInt()
+    "- 0x123".toInt()
+    "+0x123".toInt()
+    "+ 0x123".toInt()
+    "0x1_2__3___".toInt()
+    "-0x1_2__3___".toInt()
+    "+0x1_2__3___".toInt()
+    "0x0".toInt()
+    "-0x0".toInt()
+    "+0x0".toInt()
+    "0x1E_2".toInt() // test underscore removal after 'E' succeeds for toInt (fails for toFloat)
+  }
+
+  ["toInt() binary"] {
+    "0b101".toInt()
+    "0B101".toInt()
+    "-0b101".toInt()
+    "- 0b101".toInt()
+    "+0b101".toInt()
+    "+ 0b101".toInt()
+    "0b1_0__1___".toInt()
+    "-0b1_0__1___".toInt()
+    "+0b1_0__1___".toInt()
+    "0b0".toInt()
+    "-0b0".toInt()
+    "+0b0".toInt()
+  }
+
+  ["toInt() octal"] {
+    "0o123".toInt()
+    "0O123".toInt()
+    "-0o123".toInt()
+    "- 0o123".toInt()
+    "+0o123".toInt()
+    "+ 0o123".toInt()
+    "0o1_2__3___".toInt()
+    "-0o1_2__3___".toInt()
+    "+0o1_2__3___".toInt()
+    "0o0".toInt()
+    "-0o0".toInt()
+    "+0o0".toInt()
+  }
+   
+  ["toInt() error cases"] {
     module.catch(() -> "1.2".toInt())
     module.catch(() -> "9223372036854775808".toInt())
     module.catch(() -> "-9223372036854775809".toInt())
     module.catch(() -> "abc".toInt())
     module.catch(() -> "_1_000".toInt())
+    module.catch(() -> "0p0".toInt())
+    module.catch(() -> "0x-0".toInt())
+    module.catch(() -> "0x+0".toInt())
+    module.catch(() -> "0x0 + 0x1".toInt())
+    module.catch(() -> "".toInt())
+    module.catch(() -> "0o123e4".toInt())
   }
 
   ["toIntOrNull()"] {
@@ -289,6 +351,11 @@ examples {
     "-9223372036854775809".toIntOrNull()
     "abc".toIntOrNull()
     "_1_2__3___".toIntOrNull()
+    "0p0".toIntOrNull()
+    "0x-0".toIntOrNull()
+    "0x+0".toIntOrNull()
+    "0x0 + 0x1".toIntOrNull()
+    "".toIntOrNull()
   }
 
   ["toFloat()"] {

pkl-core/src/test/files/LanguageSnippetTests/output/api/reflectedDeclaration.pcf

@@ -2255,9 +2255,15 @@ alias {
             displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
           }
           docComment = """
-            Parses this string as a signed decimal (base 10) integer.
+            Parses this string as a signed integer.
             
-            Throws if this string cannot be parsed as a signed decimal integer,
+            Supports integer formats supported by Pkl:
+            * Decimal (base 10)
+            * Hexadecimal (base 16) with prefix `0x`
+            * Binary (base 2) with prefix `0b`
+            * Octal (base 8) with prefix `0o`
+            
+            Throws if this string cannot be parsed,
             or if the integer is too large to fit into [Int].
             """
           annotations = List()
@@ -2272,9 +2278,15 @@ alias {
             displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
           }
           docComment = """
-            Parses this string as a signed decimal (base 10) integer.
+            Parses this string as a signed integer.
             
-            Returns [null] if this string cannot be parsed as a signed decimal integer,
+            Supports integer formats supported by Pkl:
+            * Decimal (base 10)
+            * Hexadecimal (base 16) with prefix `0x`
+            * Binary (base 2) with prefix `0b`
+            * Octal (base 8) with prefix `0o`
+            
+            Returns [null] if this string cannot be parsed,
             or if the integer is too large to fit into [Int].
             """
           annotations = List()
@@ -3192,9 +3204,15 @@ alias {
             displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
           }
           docComment = """
-            Parses this string as a signed decimal (base 10) integer.
+            Parses this string as a signed integer.
             
-            Throws if this string cannot be parsed as a signed decimal integer,
+            Supports integer formats supported by Pkl:
+            * Decimal (base 10)
+            * Hexadecimal (base 16) with prefix `0x`
+            * Binary (base 2) with prefix `0b`
+            * Octal (base 8) with prefix `0o`
+            
+            Throws if this string cannot be parsed,
             or if the integer is too large to fit into [Int].
             """
           annotations = List()
@@ -3209,9 +3227,15 @@ alias {
             displayUri = "https://github.com/apple/pkl/blob/$commitId/stdlib/base.pkl#LXXXX"
           }
           docComment = """
-            Parses this string as a signed decimal (base 10) integer.
+            Parses this string as a signed integer.
             
-            Returns [null] if this string cannot be parsed as a signed decimal integer,
+            Supports integer formats supported by Pkl:
+            * Decimal (base 10)
+            * Hexadecimal (base 16) with prefix `0x`
+            * Binary (base 2) with prefix `0b`
+            * Octal (base 8) with prefix `0o`
+            
+            Returns [null] if this string cannot be parsed,
             or if the integer is too large to fit into [Int].
             """
           annotations = List()

pkl-core/src/test/files/LanguageSnippetTests/output/api/string.pcf

@@ -213,18 +213,76 @@ examples {
     List()
     List(97, 98, 99, 100, 101, 102, 103)
   }
-  ["toInt()"] {
+  ["toInt() decimal"] {
+    123
+    -123
+    -123
+    123
+    123
     123
     -123
     123
-    -123
     0
     0
+    0
+    -9223372036854775808
+    9223372036854775807
+  }
+  ["toInt() hex"] {
+    291
+    291
+    -291
+    -291
+    291
+    291
+    291
+    -291
+    291
+    0
+    0
+    0
+    482
+  }
+  ["toInt() binary"] {
+    5
+    5
+    -5
+    -5
+    5
+    5
+    5
+    -5
+    5
+    0
+    0
+    0
+  }
+  ["toInt() octal"] {
+    83
+    83
+    -83
+    -83
+    83
+    83
+    83
+    -83
+    83
+    0
+    0
+    0
+  }
+  ["toInt() error cases"] {
     "Cannot parse string as `Int`. String: \"1.2\""
     "Cannot parse string as `Int`. String: \"9223372036854775808\""
     "Cannot parse string as `Int`. String: \"-9223372036854775809\""
     "Cannot parse string as `Int`. String: \"abc\""
     "Cannot parse string as `Int`. String: \"_1_000\""
+    "Cannot parse string as `Int`. String: \"0p0\""
+    "Cannot parse string as `Int`. String: \"0x-0\""
+    "Cannot parse string as `Int`. String: \"0x+0\""
+    "Cannot parse string as `Int`. String: \"0x0 + 0x1\""
+    "Cannot parse string as `Int`. String: \"\""
+    "Cannot parse string as `Int`. String: \"0o123e4\""
   }
   ["toIntOrNull()"] {
     123
@@ -238,6 +296,11 @@ examples {
     null
     null
     null
+    null
+    null
+    null
+    null
+    null
   }
   ["toFloat()"] {
     0.0

pkl-parser/src/main/java/org/pkl/parser/Lexer.java

@@ -236,7 +236,7 @@ public final class Lexer {
           } else {
             throw unexpectedChar("..", ".", "...", "...?");
           }
-        } else if (lookahead >= 48 && lookahead <= 57) {
+        } else if (lookahead >= '0' && lookahead <= '9') {
           yield lexNumber(ch);
         } else {
           yield Token.DOT;
@@ -574,7 +574,7 @@ public final class Lexer {
       return Token.FLOAT;
     }
 
-    while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') {
+    while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') {
       nextChar();
     }
 
@@ -587,7 +587,7 @@ public final class Lexer {
       if (lookahead == '_') {
         throw lexError("invalidSeparatorPosition");
       }
-      if (lookahead < 48 || lookahead > 57) {
+      if (lookahead < '0' || lookahead > '9') {
         backup();
         return Token.INT;
       }
@@ -661,10 +661,10 @@ public final class Lexer {
       throw lexError("invalidSeparatorPosition");
     }
     var ch = lookahead;
-    if (!(ch >= 48 && ch <= 55)) {
+    if (!(ch >= '0' && ch <= '7')) {
       throw unexpectedChar(ch, "octal number");
     }
-    while ((ch >= 48 && ch <= 55) || ch == '_') {
+    while ((ch >= '0' && ch <= '7') || ch == '_') {
       nextChar();
       ch = lookahead;
     }
@@ -677,10 +677,10 @@ public final class Lexer {
     if (lookahead == '_') {
       throw lexError("invalidSeparatorPosition");
     }
-    if (lookahead < 48 || lookahead > 57) {
+    if (lookahead < '0' || lookahead > '9') {
       throw unexpectedChar(lookahead, "number");
     }
-    while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') {
+    while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') {
       nextChar();
     }
   }
@@ -689,7 +689,7 @@ public final class Lexer {
     if (lookahead == '_') {
       throw lexError("invalidSeparatorPosition");
     }
-    while ((lookahead >= 48 && lookahead <= 57) || lookahead == '_') {
+    while ((lookahead >= '0' && lookahead <= '9') || lookahead == '_') {
       nextChar();
     }
     if (lookahead == 'e' || lookahead == 'E') {
@@ -706,7 +706,9 @@ public final class Lexer {
   }
 
   private boolean isHex(int code) {
-    return (code >= 48 && code <= 57) || (code >= 97 && code <= 102) || (code >= 65 && code <= 70);
+    return (code >= '0' && code <= '9')
+        || (code >= 'a' && code <= 'f')
+        || (code >= 'A' && code <= 'F');
   }
 
   private static boolean isIdentifierStart(int c) {

stdlib/base.pkl

@@ -1640,15 +1640,27 @@ external class String extends Any {
   /// ```
   external function decapitalize(): String
 
-  /// Parses this string as a signed decimal (base 10) integer.
+  /// Parses this string as a signed integer.
   ///
-  /// Throws if this string cannot be parsed as a signed decimal integer,
+  /// Supports integer formats supported by Pkl:
+  /// * Decimal (base 10)
+  /// * Hexadecimal (base 16) with prefix `0x`
+  /// * Binary (base 2) with prefix `0b`
+  /// * Octal (base 8) with prefix `0o`
+  ///
+  /// Throws if this string cannot be parsed,
   /// or if the integer is too large to fit into [Int].
   external function toInt(): Int
 
-  /// Parses this string as a signed decimal (base 10) integer.
+  /// Parses this string as a signed integer.
   ///
-  /// Returns [null] if this string cannot be parsed as a signed decimal integer,
+  /// Supports integer formats supported by Pkl:
+  /// * Decimal (base 10)
+  /// * Hexadecimal (base 16) with prefix `0x`
+  /// * Binary (base 2) with prefix `0b`
+  /// * Octal (base 8) with prefix `0o`
+  ///
+  /// Returns [null] if this string cannot be parsed,
   /// or if the integer is too large to fit into [Int].
   external function toIntOrNull(): Int?