Skip to content

[table-driven-branch] Pull over the concepts from main for strtod locale issues #2193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thomasvl merged 4 commits into from
Oct 15, 2026
Merged

[table-driven-branch] Pull over the concepts from main for strtod locale issues #2193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0bfdd32
make json and text format number parsing locale-independent (#2187)
uchiha-bug-hunter Oct 15, 2026
864e7cf
Document the strtod usage constraints. (#2191)
thomasvl Oct 15, 2026
353f50c
[table-driven-branch] Reuse JSONReader/Tokenizer for quoted JSON numbers
thomasvl Oct 15, 2026
ebe5e9d
[table-driven-branch] Avoid String creation when parsing unescaped qu…
thomasvl Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
260 changes: 116 additions & 144 deletions Sources/SwiftProtobuf/JSONReader.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,6 @@
///
// -----------------------------------------------------------------------------

#if canImport(Glibc)
import Glibc
#elseif canImport(Musl)
import Musl
#elseif canImport(Android)
import Android
#elseif canImport(WASILibc)
import WASILibc
#elseif canImport(Darwin)
import Darwin
#elseif os(Windows)
import CRT
#endif

#if canImport(FoundationEssentials)
import FoundationEssentials
#else
Expand Down Expand Up @@ -150,28 +136,12 @@ struct JSONReader: TextualParser {
precondition(upperBound >= 0, "Upper bound must be non-negative")

var negative = false
var upperBound = UInt64(upperBound)
let unsigned: UInt64

if at(.string, .stringWithEscapes) {
var stringValue = try consumeString()[...]

if stringValue.first == "-" {
stringValue = stringValue.dropFirst()
negative = true
// Two's compliment always allows one more negative integer than positive.
upperBound += 1
}

unsigned = try parseUnsignedInteger(from: stringValue, upperBound: UInt64(upperBound))
unsigned = try parseQuotedInteger(isSigned: true, negative: &negative, upperBound: UInt64(upperBound))
} else {
if try consumeIfPresent(.minus) {
negative = true
// Two's compliment always allows one more negative integer than positive.
upperBound += 1
}

unsigned = try consumeUnsignedInteger(upperBound: upperBound)
unsigned = try consumeSignedMagnitude(negative: &negative, upperBound: UInt64(upperBound))
}

if negative {
Expand All @@ -184,15 +154,27 @@ struct JSONReader: TextualParser {
}
}

private mutating func consumeSignedMagnitude(
negative: inout Bool,
upperBound: UInt64
) throws -> UInt64 {
var upperBound = upperBound
if try consumeIfPresent(.minus) {
negative = true
// Two's compliment always allows one more negative integer than positive.
upperBound += 1
}
return try consumeUnsignedInteger(upperBound: upperBound)
}

/// Consumes an unsigned integer from the tokenizer.
///
/// - Parameter upperBound: The maximum value of the integer to consume. If the integer is
/// greater than `upperBound`, an error will be thrown.
mutating func consumeUnsignedInteger(upperBound: UInt64) throws -> UInt64 {
if at(.string, .stringWithEscapes) {
// Strings are allowed.
let string = try consumeString()[...]
return try parseUnsignedInteger(from: string, upperBound: upperBound)
var negative = false
return try parseQuotedInteger(isSigned: false, negative: &negative, upperBound: upperBound)
}
if at(.float) {
// Floating-point numbers are allowed as long as they correspond exactly to an integer
Expand All @@ -214,113 +196,79 @@ struct JSONReader: TextualParser {
return value
}

/// Parses a string into an unsigned integer.
///
/// - Parameters:
/// - string: The string to parse.
/// - upperBound: The maximum value of the integer to consume. If the integer is
/// greater than `upperBound`, an error will be thrown.
private mutating func parseUnsignedInteger(from string: Substring, upperBound: UInt64) throws -> UInt64 {
func valueError() -> SwiftProtobufError {
parsingError(reason: "Expected an integer in 0...\(upperBound)")
}

if string.first == "0" && string.count > 1 {
throw parsingError(reason: "Leading zeros are not allowed in integers")
}

if let value = UInt64(string) {
guard value <= upperBound else {
throw valueError()
}
return value
/// Consumes a double-quoted string, resolving any escapes, and parses it as an integer using
/// `Tokenizer` so that quoted and unquoted numbers share the exact same validation.
private mutating func parseQuotedInteger(
isSigned: Bool,
negative: inout Bool,
upperBound: UInt64
) throws -> UInt64 {
if at(.string) {
let utf8Bytes = try consumeUnescapedStringBytes()
return try parseQuotedInteger(
from: utf8Bytes,
isSigned: isSigned,
negative: &negative,
upperBound: upperBound
)
}
var stringValue = try consumeString()
return try stringValue.withUTF8 { utf8Bytes in
try parseQuotedInteger(
from: utf8Bytes,
isSigned: isSigned,
negative: &negative,
upperBound: upperBound
)
}
// Try to parse it as a Double; e.g., an exact integer in exponential form.
let doubleValue = try parseDouble(from: string)
if let uint64Value = UInt64(exactly: doubleValue), uint64Value <= upperBound {
return uint64Value
}
throw valueError()
}

/// Parses a string into a double.
///
/// - Parameter string: The string to parse.
private mutating func parseDouble(from string: Substring) throws -> Double {
// The tokenizer handles this for regular numbers, but for numbers represented as strings,
// we need to make sure there's at least one digit before and after the decimal point.
let stringWithoutMinus = (string.utf8.first == UInt8(ascii: "-")) ? string.dropFirst() : string[...]
if let dotIndex = stringWithoutMinus.firstIndex(of: ".") {
let beforeDot = stringWithoutMinus.prefix(upTo: dotIndex)
guard !beforeDot.isEmpty else {
throw parsingError(reason: "Numbers cannot start with a decimal point in JSON")
}
guard beforeDot.count == 1 || beforeDot.utf8.first != UInt8(ascii: "0") else {
throw parsingError(reason: "Leading zeros are not allowed in JSON")
}
guard !stringWithoutMinus.suffix(from: stringWithoutMinus.index(after: dotIndex)).isEmpty else {
throw parsingError(reason: "Decimal point must be followed by digits in JSON")
}
} else {
guard stringWithoutMinus.count == 1 || stringWithoutMinus.utf8.first != UInt8(ascii: "0") else {
throw parsingError(reason: "Leading zeros are not allowed in JSON")
}
}

switch string {
case "NaN":
return .nan
case "Inf", "Infinity":
return .infinity
case "-Inf", "-Infinity":
return -.infinity
default:
return try parseDoubleRaw(string)
}
private mutating func parseQuotedInteger(
from utf8Bytes: UnsafeBufferPointer<UInt8>,
isSigned: Bool,
negative: inout Bool,
upperBound: UInt64
) throws -> UInt64 {
guard let subTokenizer = try makeQuotedNumberTokenizer(utf8Bytes) else {
throw parsingError(reason: "Invalid number string")
}
let originalTokenizer = self.tokenizer
defer { self.tokenizer = originalTokenizer }
self.tokenizer = subTokenizer
let result =
isSigned
? try consumeSignedMagnitude(negative: &negative, upperBound: upperBound)
: try consumeUnsignedInteger(upperBound: upperBound)
guard complete else {
throw parsingError(reason: "Invalid number string")
}
return result
}

/// Parses a floating-point number from a string using the system's `strtod` function.
///
/// - Parameter string: The string to parse.
/// - Throws: `SwiftProtobufError` if the string does not represent a valid floating-point number.
private func parseDoubleRaw(_ string: Substring) throws -> Double {
func valueError() -> SwiftProtobufError {
parsingError(reason: "Invalid floating point value")
}
/// Consumes a `.string` token (which has no escapes) and returns the slice of UTF-8 bytes
/// between the quotes without creating a `String`.
private mutating func consumeUnescapedStringBytes() throws -> UnsafeBufferPointer<UInt8> {
let text = tokenizer.current.text
_ = try tokenizer.next()
return UnsafeBufferPointer(rebasing: text[1..<(text.count - 1)])
}

// Reject unreasonably small or large numbers, or numbers that don't start with a digit or
// minus sign (strtod isn't very strict and allows leading whitespace).
let capacity = 128
/// Creates a `Tokenizer` over the UTF-8 bytes of a quoted number string if it starts with a
/// minus sign (immediately followed by a digit) or digit and ends with a digit (so leading,
/// internal, and trailing whitespace are rejected). Returns `nil` if the bytes do not start
/// and end with digits.
private func makeQuotedNumberTokenizer(_ utf8Bytes: UnsafeBufferPointer<UInt8>) throws -> Tokenizer? {
let firstDigitIndex = (utf8Bytes.first == UInt8(ascii: "-")) ? 1 : 0
guard
let first = string.utf8.first,
string.utf8.count < capacity - 1,
first == UInt8(ascii: "-") || Tokenizer.isDigit(first)
firstDigitIndex < utf8Bytes.count,
Tokenizer.isDigit(utf8Bytes[firstDigitIndex]),
Tokenizer.isDigit(utf8Bytes.last!)
else {
throw valueError()
}
return try withUnsafeTemporaryAllocation(of: CChar.self, capacity: capacity) { buffer in
var string = string
string.withUTF8 { utf8Bytes in
utf8Bytes.withMemoryRebound(to: CChar.self) { cChars in
let (_, endIndex) = buffer.initialize(from: cChars)
buffer[endIndex] = 0
}
}

let ccharPointer = buffer.baseAddress!
var lastConsumed: UnsafeMutablePointer<CChar>? = nil
let doubleValue = strtod(ccharPointer, &lastConsumed)

// Fail if `strtod` did not consume everything we expected or if the value was not
// finite (which would happen if it was too large or too small).
guard
let lastConsumed = lastConsumed,
lastConsumed == ccharPointer + string.utf8.count, doubleValue.isFinite
else {
throw valueError()
}
return doubleValue
return nil
}
var subTokenizer = Tokenizer(buffer: utf8Bytes, mode: .json, errorCode: .jsonDecodingError)
try subTokenizer.next()
return subTokenizer
}

/// Consumes a double-quoted string, resolving any escapes.
Expand Down Expand Up @@ -351,17 +299,14 @@ struct JSONReader: TextualParser {

/// Consumes a 64-bit floating point value from the tokenizer.
mutating func consumeDouble() throws -> Double {
if at(.string, .stringWithEscapes) {
let stringValue = try consumeString()
switch stringValue {
case "NaN":
return .nan
case "Inf", "Infinity":
return .infinity
case "-Inf", "-Infinity":
return -.infinity
default:
return try parseDouble(from: stringValue[...])
if at(.string) {
let utf8Bytes = try consumeUnescapedStringBytes()
return try parseQuotedDouble(from: utf8Bytes)
}
if at(.stringWithEscapes) {
var stringValue = try consumeString()
return try stringValue.withUTF8 { utf8Bytes in
try parseQuotedDouble(from: utf8Bytes)
}
}

Expand All @@ -381,6 +326,33 @@ struct JSONReader: TextualParser {
}
}

private mutating func parseQuotedDouble(from utf8Bytes: UnsafeBufferPointer<UInt8>) throws -> Double {
guard let subTokenizer = try makeQuotedNumberTokenizer(utf8Bytes) else {
// The longest valid value here is "-Infinity" (9 bytes), so fail fast on larger input.
guard utf8Bytes.count <= 9 else {
throw parsingError(reason: "Invalid number string")
}
switch String(decoding: utf8Bytes, as: UTF8.self) {
case "NaN":
return .nan
case "Inf", "Infinity":
return .infinity
case "-Inf", "-Infinity":
return -.infinity
default:
throw parsingError(reason: "Invalid number string")
}
}
let originalTokenizer = self.tokenizer
defer { self.tokenizer = originalTokenizer }
self.tokenizer = subTokenizer
let result = try consumeDouble()
guard complete else {
throw parsingError(reason: "Invalid number string")
}
return result
}

/// Consumes an unsigned decimal integer or floating point value from the tokenizer.
mutating func consumeUnsignedDecimalAsDouble() throws -> Double {
guard at(.integer) else {
Expand Down
Loading
Loading