Add xml option

Although it's possible to do more extensive sniffing, and we might do so in the future, this is enough to pass the relevant web platform tests we're currently working on in jsdom.

Author: domenic

README.md

@@ -21,19 +21,21 @@ const htmlString = (new TextDecoder(sniffedEncoding)).decode(htmlBytes);
 
 ## Options
 
-You can pass two potential options to `htmlEncodingSniffer`:
+You can pass the following options to `htmlEncodingSniffer`:
 
 ```js
 const sniffedEncoding = htmlEncodingSniffer(htmlBytes, {
+  xml,
   transportLayerEncodingLabel,
-  defaultEncoding
+  defaultEncoding,
 });
 ```
 
-These represent two possible inputs into the [encoding sniffing algorithm](https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm):
+The `xml` option is a boolean, defaulting to `false`. If set to `true`, then we bypass the [HTML encoding sniffing algorithm](https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm) and compute the encoding based on the presence of a BOM, or the other options provided. (In the future, we may perform sniffing of the `<?xml?>` declaration, but for now that is not implemented.)
 
-- `transportLayerEncodingLabel` is an encoding label that is obtained from the "transport layer" (probably a HTTP `Content-Type` header), which overrides everything but a BOM.
-- `defaultEncoding` is the ultimate fallback encoding used if no valid encoding is supplied by the transport layer, and no encoding is sniffed from the bytes. It defaults to `"windows-1252"`, as recommended by the algorithm's table of suggested defaults for "All other locales" (including the `en` locale).
+The `transportLayerEncodingLabel` is an encoding label that is obtained from the "transport layer" (probably a HTTP `Content-Type` header), which overrides everything but a BOM.
+
+The `defaultEncoding` is the ultimate fallback encoding used if no valid encoding is supplied by the transport layer, and no encoding is sniffed from the bytes. For HTML, it defaults to `"windows-1252"`, as recommended by the algorithm's table of suggested defaults for "All other locales" (including the `en` locale). For XML, it defaults to `"UTF-8"`.
 
 ## Credits
 

lib/html-encoding-sniffer.js

@@ -2,14 +2,18 @@
 const { getBOMEncoding, labelToName } = require("@exodus/bytes/encoding-lite.js");
 
 // https://html.spec.whatwg.org/#encoding-sniffing-algorithm
-module.exports = (uint8Array, { transportLayerEncodingLabel, defaultEncoding = "windows-1252" } = {}) => {
+module.exports = (uint8Array, { xml = false, transportLayerEncodingLabel, defaultEncoding } = {}) => {
+  if (defaultEncoding === undefined) {
+    defaultEncoding = xml ? "UTF-8" : "windows-1252";
+  }
+
   let encoding = labelToName(getBOMEncoding(uint8Array));
 
   if (encoding === null && transportLayerEncodingLabel !== undefined) {
     encoding = labelToName(transportLayerEncodingLabel);
   }
 
-  if (encoding === null) {
+  if (encoding === null && !xml) {
     encoding = prescanMetaCharset(uint8Array);
   }
 

test/tests.js

@@ -124,3 +124,104 @@ for (const file of fs.readdirSync(path.resolve(__dirname, "fixtures/utf"))) {
     });
   });
 }
+
+describe("xml: true", () => {
+  describe("BOM detection", () => {
+    for (const file of fs.readdirSync(path.resolve(__dirname, "fixtures/bom"))) {
+      const buffer = read(`fixtures/bom/${file}`);
+      const desiredEncoding = path.basename(file, ".html");
+
+      it(`should sniff ${file} as ${desiredEncoding}`, () => {
+        const sniffedEncoding = htmlEncodingSniffer(buffer, { xml: true });
+
+        assert.strictEqual(sniffedEncoding, desiredEncoding);
+      });
+
+      it(`should sniff ${file} as ${desiredEncoding}, given overriding options`, () => {
+        const sniffedEncoding = htmlEncodingSniffer(buffer, {
+          xml: true,
+          transportLayerEncodingLabel: "windows-1252",
+          defaultEncoding: "ISO-8859-1"
+        });
+
+        assert.strictEqual(sniffedEncoding, desiredEncoding);
+      });
+    }
+  });
+
+  describe("UTF-32 BOMs (not recognized, should fall back to default)", () => {
+    it("should ignore UTF-32BE BOM and return UTF-8", () => {
+      // UTF-32BE BOM: 00 00 FE FF
+      const buffer = new Uint8Array([0x00, 0x00, 0xFE, 0xFF, 0x3C, 0x3F, 0x78, 0x6D, 0x6C]);
+      const sniffedEncoding = htmlEncodingSniffer(buffer, { xml: true });
+
+      assert.strictEqual(sniffedEncoding, "UTF-8");
+    });
+
+    it("should detect UTF-32LE BOM as UTF-16LE (since FF FE prefix matches)", () => {
+      // UTF-32LE BOM: FF FE 00 00 — but FF FE is also UTF-16LE BOM
+      const buffer = new Uint8Array([0xFF, 0xFE, 0x00, 0x00, 0x3C, 0x3F, 0x78, 0x6D, 0x6C]);
+      const sniffedEncoding = htmlEncodingSniffer(buffer, { xml: true });
+
+      assert.strictEqual(sniffedEncoding, "UTF-16LE");
+    });
+
+    it("should ignore UTF-32BE BOM and use transport layer encoding if provided", () => {
+      const buffer = new Uint8Array([0x00, 0x00, 0xFE, 0xFF, 0x3C, 0x3F, 0x78, 0x6D, 0x6C]);
+      const sniffedEncoding = htmlEncodingSniffer(buffer, {
+        xml: true,
+        transportLayerEncodingLabel: "KOI8-R"
+      });
+
+      assert.strictEqual(sniffedEncoding, "KOI8-R");
+    });
+  });
+
+  describe("meta charset ignored", () => {
+    it("should ignore meta charset and return UTF-8 default", () => {
+      const buffer = read("fixtures/normal/charset_KOI8-R.html");
+      const sniffedEncoding = htmlEncodingSniffer(buffer, { xml: true });
+
+      assert.strictEqual(sniffedEncoding, "UTF-8");
+    });
+
+    it("should ignore meta charset but use transport layer encoding", () => {
+      const buffer = read("fixtures/normal/charset_KOI8-R.html");
+      const sniffedEncoding = htmlEncodingSniffer(buffer, {
+        xml: true,
+        transportLayerEncodingLabel: "ISO-8859-2"
+      });
+
+      assert.strictEqual(sniffedEncoding, "ISO-8859-2");
+    });
+
+    it("should ignore meta charset but use custom default encoding", () => {
+      const buffer = read("fixtures/normal/charset_KOI8-R.html");
+      const sniffedEncoding = htmlEncodingSniffer(buffer, {
+        xml: true,
+        defaultEncoding: "windows-1252"
+      });
+
+      assert.strictEqual(sniffedEncoding, "windows-1252");
+    });
+  });
+
+  describe("default encoding", () => {
+    it("should default to UTF-8 for XML", () => {
+      const buffer = read("fixtures/no-result/no-indicators_windows-1252.html");
+      const sniffedEncoding = htmlEncodingSniffer(buffer, { xml: true });
+
+      assert.strictEqual(sniffedEncoding, "UTF-8");
+    });
+
+    it("should allow overriding the default encoding", () => {
+      const buffer = read("fixtures/no-result/no-indicators_windows-1252.html");
+      const sniffedEncoding = htmlEncodingSniffer(buffer, {
+        xml: true,
+        defaultEncoding: "ISO-8859-1"
+      });
+
+      assert.strictEqual(sniffedEncoding, "ISO-8859-1");
+    });
+  });
+});