string_decoder: decode UTF-8 via StringBytes::Encode

StringDecoder used v8::String::NewFromUtf8() for UTF-8, while Buffer#toString() goes through StringBytes::Encode(), which has simdutf-backed ASCII, Latin-1 and UTF-16 paths and only falls back to NewFromUtf8() for input that contains invalid sequences. Route the decoder through the same function, so streams with setEncoding('utf8') and readline decode at the same speed as Buffer#toString(). U+FFFD replacement is unchanged because invalid input still ends up in NewFromUtf8(), and the ERR_STRING_TOO_LONG check is kept explicit so over-long input fails as before. benchmark/string_decoder/string-decoder.js (encoding=utf8) and a readline-over-pipe workload improve by 2-3x for chunks >= 1 KiB; 64 KiB newline-delimited JSON round trips over child stdio improve by ~30% on the reading side alone. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65324 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day> Reviewed-By: Daniel Lemire <daniel@lemire.me> Reviewed-By: James M Snell <jasnell@gmail.com>
aduh95 · Aug 26, 2026 · Aug 3, 2026 · Aug 9, 2026 · Aug 10, 2026 · Aug 10, 2026
commit f33dba704d35c4f61b3d574d781e04e586f3480b
diff --git a/src/string_decoder.cc b/src/string_decoder.cc
@@ -28,23 +28,21 @@ MaybeLocal<String> MakeString(Isolate* isolate,
                               const char* data,
                               size_t length,
                               enum encoding encoding) {
-  MaybeLocal<Value> ret;
-  if (encoding == UTF8) {
-    MaybeLocal<String> utf8_string;
-    if (length <= static_cast<size_t>(v8::String::kMaxLength)) {
-      utf8_string = String::NewFromUtf8(
-          isolate, data, v8::NewStringType::kNormal, length);
-    }
-    if (utf8_string.IsEmpty()) {
-      isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate));
-      return MaybeLocal<String>();
-    } else {
-      return utf8_string;
-    }
-  } else {
-    ret = StringBytes::Encode(isolate, data, length, encoding);
+  // StringBytes::Encode() would report an over-long UTF-8 input as
+  // ERR_BUFFER_TOO_LARGE (or clamp it); keep reporting it the way this
+  // decoder always has.
+  if (encoding == UTF8 && length > static_cast<size_t>(v8::String::kMaxLength))
+      [[unlikely]] {
+    isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate));
+    return MaybeLocal<String>();
   }
 
+  // For UTF-8 this takes the simdutf-backed ASCII / Latin-1 / UTF-16 paths and
+  // only falls back to v8::String::NewFromUtf8() (the previous unconditional
+  // path here) for input containing invalid sequences, so U+FFFD replacement
+  // is unchanged.
+  MaybeLocal<Value> ret = StringBytes::Encode(isolate, data, length, encoding);
+
   if (ret.IsEmpty()) {
     return {};
   }

diff --git a/test/parallel/test-string-decoder-utf8-large.js b/test/parallel/test-string-decoder-utf8-large.js
@@ -0,0 +1,81 @@
+'use strict';
+// The UTF-8 StringDecoder shares its byte->string conversion with
+// Buffer#toString(): ASCII, Latin-1-representable and general inputs take
+// different (SIMD) paths depending on content and size, and invalid input
+// falls back to a replacing decoder. This test pins the decoder's output for
+// inputs that cross those size thresholds, for chunkings that split multibyte
+// sequences, and for invalid bytes embedded in otherwise large valid input.
+require('../common');
+const assert = require('assert');
+const { StringDecoder } = require('string_decoder');
+
+function decodeInChunks(buf, chunkSize) {
+  const decoder = new StringDecoder('utf8');
+  let out = '';
+  for (let i = 0; i < buf.length; i += chunkSize) {
+    out += decoder.write(buf.subarray(i, i + chunkSize));
+  }
+  return out + decoder.end();
+}
+
+function check(str, label) {
+  const buf = Buffer.from(str, 'utf8');
+  // Sanity: the expectation itself round-trips.
+  assert.strictEqual(buf.toString('utf8'), str, `${label}: toString`);
+  for (const chunkSize of [1, 2, 3, 4, 5, 7, 31, 32, 33, 255, 256, 257,
+                           4095, 4096, 65536, buf.length]) {
+    if (chunkSize > buf.length) continue;
+    // Keep the test fast: byte-sized chunks only for the smaller inputs.
+    if (buf.length > 100_000 && chunkSize < 4095) continue;
+    assert.strictEqual(decodeInChunks(buf, chunkSize), str,
+                       `${label}: chunkSize=${chunkSize}`);
+  }
+}
+
+const sizes = [31, 32, 33, 255, 256, 257, 4096, 70000, (1 << 20) + 5];
+for (const size of sizes) {
+  check('a'.repeat(size), `ascii ${size}`);
+  // Latin-1 range only (one-byte string in V8, two bytes each in UTF-8).
+  check('é'.repeat(size), `latin1 ${size}`);
+  // ASCII with a single Latin-1 character at the end / start.
+  check('a'.repeat(size - 1) + 'ÿ', `ascii+latin1 tail ${size}`);
+  check('Ä' + 'a'.repeat(size - 1), `latin1 head+ascii ${size}`);
+  // BMP beyond Latin-1 (three-byte sequences).
+  check('日'.repeat(size), `cjk ${size}`);
+  // Mixed, including astral plane characters (surrogate pairs, 4 bytes).
+  check(('abé日\u{1F600}').repeat(Math.ceil(size / 6)), `mixed ${size}`);
+}
+
+// Invalid bytes inside otherwise valid input of every size class must still be
+// replaced with U+FFFD exactly as before, regardless of chunking.
+for (const size of [8, 40, 300, 5000, (1 << 20) + 5]) {
+  const valid = Buffer.from('a'.repeat(size));
+  for (const bad of [[0xff], [0xc0, 0xaf], [0xe2, 0x28, 0xa1],
+                     [0xed, 0xa0, 0x80] /* encoded surrogate */,
+                     [0xf0, 0x9f, 0x98] /* truncated 4-byte */]) {
+    const buf = Buffer.concat([valid, Buffer.from(bad), valid]);
+    const expected = buf.toString('utf8');
+    assert.ok(expected.includes('�'), `size=${size} bad=${bad}`);
+    for (const chunkSize of [1, 3, 64, size, size + 1, buf.length]) {
+      if (buf.length > 100_000 && chunkSize < size) continue;
+      assert.strictEqual(decodeInChunks(buf, chunkSize), expected,
+                         `invalid ${bad} in ${size}, chunkSize=${chunkSize}`);
+    }
+  }
+}
+
+// A lone continuation / lead byte split across the size classes at the very
+// end is buffered by the decoder and flushed as U+FFFD by end().
+{
+  const decoder = new StringDecoder('utf8');
+  const big = Buffer.concat([Buffer.from('a'.repeat(300)), Buffer.from([0xe2, 0x82])]);
+  assert.strictEqual(decoder.write(big), 'a'.repeat(300));
+  assert.strictEqual(decoder.end(), '�');
+}
+{
+  const decoder = new StringDecoder('utf8');
+  const big = Buffer.concat([Buffer.from('é'.repeat(300)), Buffer.from([0xe2, 0x82])]);
+  assert.strictEqual(decoder.write(big), 'é'.repeat(300));
+  assert.strictEqual(decoder.write(Buffer.from([0xac])), '€');
+  assert.strictEqual(decoder.end(), '');
+}