From bf8ba05d33b1fe5f9072db697b3c8acb56cf5148 Mon Sep 17 00:00:00 2001 From: Shubham Padkonde Date: Fri, 2 Oct 2026 17:49:07 +0530 Subject: [PATCH] fix: decode textual bytes from Unicode code points --- bytes.go | 14 +++++++++++++- bytes_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/bytes.go b/bytes.go index cf971b05..73254ee4 100644 --- a/bytes.go +++ b/bytes.go @@ -118,10 +118,13 @@ func bytesNativeFromTextual(buf []byte) (interface{}, []byte, error) { // necessarily represent text. Therefore, Avro bytes are not // encoded in UTF-16. Each \u is followed by 4 hexadecimal // digits, the first and second of which must be 0. - v, err := parseUint64FromHexSlice(buf[i+3 : i+5]) + v, err := parseUint64FromHexSlice(buf[i+1 : i+5]) if err != nil { return nil, nil, fmt.Errorf("cannot decode textual bytes: %s", err) } + if v > 255 { + return nil, nil, fmt.Errorf("cannot decode textual bytes: code point out of range: U+%04X", v) + } i += 4 // absorb 4 characters: one 'u' and three of the digits newBytes = append(newBytes, byte(v)) continue @@ -136,6 +139,15 @@ func bytesNativeFromTextual(buf []byte) (interface{}, []byte, error) { if b == '"' { return newBytes, buf[i+1:], nil } + if b >= utf8.RuneSelf { + r, width := utf8.DecodeRune(buf[i:]) + if r == utf8.RuneError || r > 255 { + return nil, nil, fmt.Errorf("cannot decode textual bytes: invalid byte code point: %U", r) + } + newBytes = append(newBytes, byte(r)) + i += width - 1 + continue + } newBytes = append(newBytes, b) } return nil, nil, fmt.Errorf("cannot decode textual bytes: expected final \"; found: %#U", buf[buflen-1]) diff --git a/bytes_test.go b/bytes_test.go index b2c4c8c0..9a970961 100644 --- a/bytes_test.go +++ b/bytes_test.go @@ -199,3 +199,50 @@ func TestStringCodecAcceptsBytes(t *testing.T) { testTextEncodePass(t, schema, []byte("abcd"), []byte(`"abcd"`)) }) } + +func TestBytesTextUnicodeCodePoints(t *testing.T) { + codec, err := NewCodec(`"bytes"`) + if err != nil { + t.Fatal(err) + } + for value := 0; value <= 255; value++ { + text, err := json.Marshal(string(rune(value))) + if err != nil { + t.Fatal(err) + } + decoded, remaining, err := codec.NativeFromTextual(append(text, ' ')) + if err != nil { + t.Fatalf("code point %d: %v", value, err) + } + got := decoded.([]byte) + if len(got) != 1 || got[0] != byte(value) || string(remaining) != " " { + t.Errorf("code point %d: got %v with remaining %q", value, got, remaining) + } + } +} + +func TestBytesTextRejectsOutOfRangeCodePoints(t *testing.T) { + for _, text := range []string{`"\u0100"`, `"\uffff"`, `"\uGGff"`, `"Ā"`, `"😂"`, "\"\xff\""} { + _, _, err := bytesNativeFromTextual([]byte(text)) + if err == nil { + t.Errorf("expected error for %q", text) + } + } +} + +func TestFixedTextUnicodeCodePoint(t *testing.T) { + codec, err := NewCodec(`{"type":"fixed","name":"single","size":1}`) + if err != nil { + t.Fatal(err) + } + for _, text := range []string{`"ÿ"`, `"\u00ff"`} { + decoded, rest, err := codec.NativeFromTextual([]byte(text)) + if err != nil { + t.Fatal(err) + } + value := decoded.([]byte) + if len(value) != 1 || value[0] != 255 || len(rest) != 0 { + t.Fatalf("%s decoded to %v, remaining %q", text, value, rest) + } + } +}