From 7188352c5c76c9df4dbc440be45ede43cb6692a9 Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Sun, 9 Aug 2026 22:10:35 +0200 Subject: [PATCH 1/5] initial processed input rewrite --- .../jfiveparse/ProcessedInputStream2.java | 44 +++++++++++++++++++ .../jfiveparse/ProcessedInputStream2Test.java | 20 +++++++++ 2 files changed, 64 insertions(+) create mode 100644 src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java create mode 100644 src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java diff --git a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java new file mode 100644 index 0000000..07f2c5a --- /dev/null +++ b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java @@ -0,0 +1,44 @@ +package ch.digitalfondue.jfiveparse; + +import java.io.IOException; +import java.io.Reader; +import java.nio.CharBuffer; + +/// goal of this class is +/// - replace the processed input stream +/// - have a single input type (a Reader) +/// - do buffering instead of calling read n times +final class ProcessedInputStream2 { + + private final Reader reader; + private final CharBuffer charBuffer = CharBuffer.allocate(1024); + + ProcessedInputStream2(Reader reader) { + this.reader = reader; + fillBuffer(); + } + + private boolean fillBuffer() { + try { + charBuffer.clear(); + var count = reader.read(charBuffer); + // after reading, handle \r\n + // + charBuffer.position(0); + charBuffer.limit(Math.max(count, 0)); + return count < 1; + } catch (IOException e) { + throw new ParserException(e); + } + } + + int read() { + var remaining = charBuffer.remaining(); + if (remaining > 0) { + return charBuffer.get(); + } else { + var isEof = fillBuffer(); + return isEof && charBuffer.remaining() == 0 ? Characters.EOF : charBuffer.get(); + } + } +} diff --git a/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java b/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java new file mode 100644 index 0000000..3afc8e4 --- /dev/null +++ b/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java @@ -0,0 +1,20 @@ +package ch.digitalfondue.jfiveparse; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.StringReader; + +class ProcessedInputStream2Test { + + @Test + void checkRead() { + var is2 = new ProcessedInputStream2(new StringReader("hello world")); + int c; + StringBuilder sb = new StringBuilder(); + while ((c = is2.read()) != Characters.EOF) { + sb.append((char) c); + } + Assertions.assertEquals("hello world", sb.toString()); + } +} From 257354ea83d98f3beaaaf6cc3d0acc0a571c33a7 Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Sun, 9 Aug 2026 22:28:55 +0200 Subject: [PATCH 2/5] fix --- .../ch/digitalfondue/jfiveparse/ProcessedInputStream2.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java index 07f2c5a..f6d5158 100644 --- a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java +++ b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java @@ -22,8 +22,11 @@ private boolean fillBuffer() { try { charBuffer.clear(); var count = reader.read(charBuffer); - // after reading, handle \r\n - // + // after reading, handle "CR LF" pair and single "CR" + // see https://infra.spec.whatwg.org/#normalize-newlines + // note, we need to take care of the special case where CR is at the end of + // the buffer, in this case, we replace it with LF, and at the next fill, if the first + // char is a LF, we advance position by 1 thus skipping it charBuffer.position(0); charBuffer.limit(Math.max(count, 0)); return count < 1; From beb348303b08c2c03cecc315facfd90bab0bdbfc Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Mon, 10 Aug 2026 19:43:08 +0200 Subject: [PATCH 3/5] test --- .../digitalfondue/jfiveparse/ProcessedInputStream2.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java index f6d5158..2588fb5 100644 --- a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java +++ b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java @@ -2,7 +2,10 @@ import java.io.IOException; import java.io.Reader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.CharBuffer; +import java.nio.LongBuffer; /// goal of this class is /// - replace the processed input stream @@ -11,7 +14,10 @@ final class ProcessedInputStream2 { private final Reader reader; - private final CharBuffer charBuffer = CharBuffer.allocate(1024); + private final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(1024 * Character.BYTES); + private final CharBuffer charBuffer = byteBuffer.asCharBuffer(); + //private final LongBuffer longBuffer = byteBuffer.asLongBuffer(); + //private final CharBuffer charBuffer = CharBuffer.allocate(1024); ProcessedInputStream2(Reader reader) { this.reader = reader; From 07d7c347b2f654da8b6db5c372f29fea70fc8d8e Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Mon, 10 Aug 2026 21:32:51 +0200 Subject: [PATCH 4/5] muh --- .../jfiveparse/ProcessedInputStream2.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java index 2588fb5..7f3290c 100644 --- a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java +++ b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java @@ -16,7 +16,7 @@ final class ProcessedInputStream2 { private final Reader reader; private final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(1024 * Character.BYTES); private final CharBuffer charBuffer = byteBuffer.asCharBuffer(); - //private final LongBuffer longBuffer = byteBuffer.asLongBuffer(); + private final LongBuffer longBuffer = byteBuffer.asLongBuffer(); //private final CharBuffer charBuffer = CharBuffer.allocate(1024); ProcessedInputStream2(Reader reader) { @@ -34,7 +34,26 @@ private boolean fillBuffer() { // the buffer, in this case, we replace it with LF, and at the next fill, if the first // char is a LF, we advance position by 1 thus skipping it charBuffer.position(0); - charBuffer.limit(Math.max(count, 0)); + + var limit = Math.max(count, 0); + /* + for (int i = 0; i < limit/4; i++) { + // TODO: here can do SWAR for CR LF search + // and rewrite + long packed = longBuffer.get(i); + char c1 = (char) (packed >>> 48); + char c2 = (char) (packed >>> 32); + char c3 = (char) (packed >>> 16); + char c4 = (char) (packed); + String unpackedString = new String(new char[]{c1, c2, c3, c4}); + System.out.println(unpackedString); + } + for (int i = limit - (limit % 4); i < limit; i++) { + System.out.println(charBuffer.get(i)); + } + */ + + charBuffer.limit(limit); return count < 1; } catch (IOException e) { throw new ParserException(e); From caa99ad33b48cf6701dd4dac135ed6931d0422cf Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Tue, 11 Aug 2026 00:18:23 +0200 Subject: [PATCH 5/5] muh2 --- .../jfiveparse/ProcessedInputStream2.java | 18 ++++++++++++------ .../jfiveparse/ProcessedInputStream2Test.java | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java index 7f3290c..71884fe 100644 --- a/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java +++ b/src/main/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2.java @@ -14,13 +14,15 @@ final class ProcessedInputStream2 { private final Reader reader; - private final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(1024 * Character.BYTES); - private final CharBuffer charBuffer = byteBuffer.asCharBuffer(); - private final LongBuffer longBuffer = byteBuffer.asLongBuffer(); - //private final CharBuffer charBuffer = CharBuffer.allocate(1024); + private final ByteBuffer byteBuffer; + private final CharBuffer charBuffer; + private final LongBuffer longBuffer; - ProcessedInputStream2(Reader reader) { + ProcessedInputStream2(Reader reader, int capacity) { this.reader = reader; + this.byteBuffer = ByteBuffer.allocateDirect(capacity * Character.BYTES); + this.charBuffer = byteBuffer.asCharBuffer(); + this.longBuffer = byteBuffer.asLongBuffer(); fillBuffer(); } @@ -39,7 +41,11 @@ private boolean fillBuffer() { /* for (int i = 0; i < limit/4; i++) { // TODO: here can do SWAR for CR LF search - // and rewrite + // and rewrite, note the code is quite complex, + // maybe we can simply do a CR ad rewrite byte by byte, + // most likely CR is not that present anyway. + // what we must take care of is if the CR/LF is at the boundary, we will need to check if LF + // is present long packed = longBuffer.get(i); char c1 = (char) (packed >>> 48); char c2 = (char) (packed >>> 32); diff --git a/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java b/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java index 3afc8e4..f27dc95 100644 --- a/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java +++ b/src/test/java/ch/digitalfondue/jfiveparse/ProcessedInputStream2Test.java @@ -9,7 +9,7 @@ class ProcessedInputStream2Test { @Test void checkRead() { - var is2 = new ProcessedInputStream2(new StringReader("hello world")); + var is2 = new ProcessedInputStream2(new StringReader("hello world"), 4); int c; StringBuilder sb = new StringBuilder(); while ((c = is2.read()) != Characters.EOF) {