src: implement Windows-1252 encoding support and update related tests

Author: mertcanaltin

lib/internal/encoding.js

@@ -28,7 +28,7 @@ const kEncoding = Symbol('encoding');
 const kDecoder = Symbol('decoder');
 const kFatal = Symbol('kFatal');
 const kUTF8FastPath = Symbol('kUTF8FastPath');
-const kLatin1FastPath = Symbol('kLatin1FastPath');
+const kWindows1252FastPath = Symbol('kWindows1252FastPath');
 const kIgnoreBOM = Symbol('kIgnoreBOM');
 
 const {
@@ -55,7 +55,7 @@ const {
   encodeIntoResults,
   encodeUtf8String,
   decodeUTF8,
-  decodeLatin1,
+  decodeWindows1252,
 } = binding;
 
 const { Buffer } = require('buffer');
@@ -420,10 +420,10 @@ function makeTextDecoderICU() {
       this[kFatal] = Boolean(options?.fatal);
       // Only support fast path for UTF-8.
       this[kUTF8FastPath] = enc === 'utf-8';
-      this[kLatin1FastPath] = enc === 'windows-1252';
+      this[kWindows1252FastPath] = enc === 'windows-1252';
       this[kHandle] = undefined;
 
-      if (!this[kUTF8FastPath] && !this[kLatin1FastPath]) {
+      if (!this[kUTF8FastPath] && !this[kWindows1252FastPath]) {
         this.#prepareConverter();
       }
     }
@@ -440,14 +440,14 @@ function makeTextDecoderICU() {
       validateDecoder(this);
 
       this[kUTF8FastPath] &&= !(options?.stream);
-      this[kLatin1FastPath] &&= !(options?.stream);
+      this[kWindows1252FastPath] &&= !(options?.stream);
 
       if (this[kUTF8FastPath]) {
         return decodeUTF8(input, this[kIgnoreBOM], this[kFatal]);
       }
 
-      if (this[kLatin1FastPath]) {
-        return decodeLatin1(input, this[kIgnoreBOM], this[kFatal]);
+      if (this[kWindows1252FastPath]) {
+        return decodeWindows1252(input, this[kIgnoreBOM], this[kFatal]);
       }
 
       this.#prepareConverter();

src/encoding_binding.cc

@@ -222,6 +222,8 @@ void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
   SetMethodNoSideEffect(isolate, target, "toASCII", ToASCII);
   SetMethodNoSideEffect(isolate, target, "toUnicode", ToUnicode);
   SetMethodNoSideEffect(isolate, target, "decodeLatin1", DecodeLatin1);
+  SetMethodNoSideEffect(
+      isolate, target, "decodeWindows1252", DecodeWindows1252);
 }
 
 void BindingData::CreatePerContextProperties(Local<Object> target,
@@ -240,6 +242,7 @@ void BindingData::RegisterTimerExternalReferences(
   registry->Register(ToASCII);
   registry->Register(ToUnicode);
   registry->Register(DecodeLatin1);
+  registry->Register(DecodeWindows1252);
 }
 
 void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
@@ -288,6 +291,77 @@ void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
   }
 }
 
+void BindingData::DecodeWindows1252(const FunctionCallbackInfo<Value>& args) {
+  Environment* env = Environment::GetCurrent(args);
+
+  CHECK_GE(args.Length(), 1);
+  if (!(args[0]->IsArrayBuffer() || args[0]->IsSharedArrayBuffer() ||
+        args[0]->IsArrayBufferView())) {
+    return node::THROW_ERR_INVALID_ARG_TYPE(
+        env->isolate(),
+        "The \"input\" argument must be an instance of ArrayBuffer, "
+        "SharedArrayBuffer, or ArrayBufferView.");
+  }
+
+  bool ignore_bom = args[1]->IsTrue();
+  bool has_fatal = args[2]->IsTrue();
+
+  ArrayBufferViewContents<uint8_t> buffer(args[0]);
+  const uint8_t* data = buffer.data();
+  size_t length = buffer.length();
+
+  if (ignore_bom && length > 0 && data[0] == 0xFF) {
+    data++;
+    length--;
+  }
+
+  if (length == 0) {
+    return args.GetReturnValue().SetEmptyString();
+  }
+
+  // Windows-1252 specific mapping for bytes 128-159
+  // These differ from Latin-1/ISO-8859-1
+  static const uint16_t windows1252_mapping[32] = {
+    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,  // 80-87
+    0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,  // 88-8F
+    0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,  // 90-97
+    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178   // 98-9F
+  };
+
+  std::string result;
+  result.reserve(length * 3);  // Reserve space for UTF-8 output
+
+  for (size_t i = 0; i < length; i++) {
+    uint8_t byte = data[i];
+    uint32_t codepoint;
+
+    // Check if byte is in the special Windows-1252 range (128-159)
+    if (byte >= 0x80 && byte <= 0x9F) {
+      codepoint = windows1252_mapping[byte - 0x80];
+    } else {
+      // For all other bytes, Windows-1252 is identical to Latin-1
+      codepoint = byte;
+    }
+
+    // Convert codepoint to UTF-8
+    if (codepoint < 0x80) {
+      result.push_back(static_cast<char>(codepoint));
+    } else if (codepoint < 0x800) {
+      result.push_back(static_cast<char>(0xC0 | (codepoint >> 6)));
+      result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
+    } else {
+      result.push_back(static_cast<char>(0xE0 | (codepoint >> 12)));
+      result.push_back(static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F)));
+      result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
+    }
+  }
+
+  Local<Value> ret;
+  if (ToV8Value(env->context(), result, env->isolate()).ToLocal(&ret)) {
+    args.GetReturnValue().Set(ret);
+  }
+}
+
 }  // namespace encoding_binding
 }  // namespace node
 

src/encoding_binding.h

@@ -32,6 +32,8 @@ class BindingData : public SnapshotableObject {
   static void EncodeUtf8String(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void DecodeUTF8(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void DecodeLatin1(const v8::FunctionCallbackInfo<v8::Value>& args);
+  static void DecodeWindows1252(
+      const v8::FunctionCallbackInfo<v8::Value>& args);
 
   static void ToASCII(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void ToUnicode(const v8::FunctionCallbackInfo<v8::Value>& args);

test/parallel/test-internal-encoding-binding.js

@@ -46,3 +46,25 @@ const binding = internalBinding('encoding_binding');
   const buf = Uint8Array.from([0xFE, 0xFF, 0xC1, 0xE9, 0xF3]);
   assert.strictEqual(binding.decodeLatin1(buf, true, true), 'þÿÁéó');
 }
+
+// Windows-1252 specific tests
+{
+  // Test Windows-1252 special characters in 128-159 range
+  // These differ from Latin-1
+  assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x80), false, false), '€');
+  assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x82), false, false), '‚');
+  assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x83), false, false), 'ƒ');
+  assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x9F), false, false), 'Ÿ');
+}
+
+{
+  // Test Windows-1252 characters outside 128-159 range (same as Latin-1)
+  const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
+  assert.strictEqual(binding.decodeWindows1252(buf, false, false), 'Áéó');
+}
+
+{
+  // Empty input
+  const buf = Uint8Array.from([]);
+  assert.strictEqual(binding.decodeWindows1252(buf, false, false), '');
+}

test/parallel/test-util-text-decoder.js

@@ -15,3 +15,49 @@ test('TextDecoder correctly decodes windows-1252 encoded data', { skip: !common.
 
   assert.strictEqual(decodedString, expectedString);
 });
+
+// Test for the difference between Latin1 and Windows-1252 in the 128-159
+// range
+// Ref: https://github.com/nodejs/node/issues/60888
+test('TextDecoder correctly decodes windows-1252 special characters in ' +
+     '128-159 range', { skip: !common.hasIntl }, () => {
+  const decoder = new TextDecoder('windows-1252');
+
+  // Test specific characters that differ between Latin1 and Windows-1252.
+  // € Euro sign
+  assert.strictEqual(decoder.decode(Uint8Array.of(128)).codePointAt(0),
+                     8364);
+  // ‚ Single low-9 quotation mark
+  assert.strictEqual(decoder.decode(Uint8Array.of(130)).codePointAt(0),
+                     8218);
+  // Latin small letter f with hook (ƒ)
+  assert.strictEqual(decoder.decode(Uint8Array.of(131)).codePointAt(0),
+                     402);
+  // Ÿ Latin capital letter Y with diaeresis
+  assert.strictEqual(decoder.decode(Uint8Array.of(159)).codePointAt(0),
+                     376);
+
+  // Test the full range to ensure no character is treated as Latin1
+  // Directly.
+  const expectedMappings = [
+    [128, 8364], [129, 129], [130, 8218], [131, 402], [132, 8222],
+    [133, 8230], [134, 8224], [135, 8225], [136, 710], [137, 8240],
+    [138, 352], [139, 8249], [140, 338], [141, 141], [142, 381],
+    [143, 143], [144, 144], [145, 8216], [146, 8217], [147, 8220],
+    [148, 8221], [149, 8226], [150, 8211], [151, 8212], [152, 732],
+    [153, 8482], [154, 353], [155, 8250], [156, 339], [157, 157],
+    [158, 382], [159, 376],
+  ];
+
+  for (const [byte, expectedCodePoint] of expectedMappings) {
+    const result = decoder.decode(Uint8Array.of(byte));
+    const actualCodePoint = result.codePointAt(0);
+    assert.strictEqual(
+      actualCodePoint,
+      expectedCodePoint,
+      `Byte 0x${byte.toString(16)} should decode to ` +
+      `U+${expectedCodePoint.toString(16)} but got ` +
+      `U+${actualCodePoint.toString(16)}`
+    );
+  }
+});

typings/internalBinding/encoding_binding.d.ts

@@ -5,4 +5,5 @@ export interface EncodingBinding {
   toASCII(input: string): string;
   toUnicode(input: string): string;
   decodeLatin1(buffer: ArrayBufferView | ArrayBuffer | SharedArrayBuffer, ignoreBOM?: boolean, hasFatal?: boolean): string;
+  decodeWindows1252(buffer: ArrayBufferView | ArrayBuffer | SharedArrayBuffer, ignoreBOM?: boolean, hasFatal?: boolean): string;
 }