test: pull enconding WPT test fixtures

PR-URL: #25321
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>

Author: joyeecheung

test/fixtures/wpt/README.md

@@ -10,10 +10,11 @@ See [test/wpt](../../wpt/README.md) for information on how these tests are run.
 
 Last update:
 
-- resources: https://github.com/web-platform-tests/wpt/tree/679a364421/resources
-- interfaces: https://github.com/web-platform-tests/wpt/tree/db7f86289e/interfaces
 - console: https://github.com/web-platform-tests/wpt/tree/9786a4b131/console
+- encoding: https://github.com/web-platform-tests/wpt/tree/a093a659ed/encoding
 - url: https://github.com/web-platform-tests/wpt/tree/75b0f336c5/url
+- resources: https://github.com/web-platform-tests/wpt/tree/679a364421/resources
+- interfaces: https://github.com/web-platform-tests/wpt/tree/712c9f275e/interfaces
 
 [Web Platform Tests]: https://github.com/web-platform-tests/wpt
 [`git node wpt`]: https://github.com/nodejs/node-core-utils/blob/master/docs/git-node.md#git-node-wpt

test/fixtures/wpt/encoding/META.yml

@@ -0,0 +1,4 @@
+spec: https://encoding.spec.whatwg.org/
+suggested_reviewers:
+ - inexorabletash
+ - annevk

test/fixtures/wpt/encoding/api-basics.any.js

@@ -0,0 +1,52 @@
+// META: title=Encoding API: Basics
+
+test(function(){
+assert_equals((newTextEncoder).encoding,'utf-8','default encoding is utf-8');
+assert_equals((newTextDecoder).encoding,'utf-8','default encoding is utf-8');
+},'Default encodings');
+
+test(function(){
+assert_array_equals(newTextEncoder().encode(),[],'input default should be empty string')
+assert_array_equals(newTextEncoder().encode(undefined),[],'input default should be empty string')
+},'Default inputs');
+
+
+functiontestDecodeSample(encoding,string,bytes){
+test(function(){
+assert_equals(newTextDecoder(encoding).decode(newUint8Array(bytes)),string);
+assert_equals(newTextDecoder(encoding).decode(newUint8Array(bytes).buffer),string);
+},'Decode sample: '+encoding);
+}
+
+// z (ASCII U+007A), cent (Latin-1 U+00A2), CJK water (BMP U+6C34),
+// G-Clef (non-BMP U+1D11E), PUA (BMP U+F8FF), PUA (non-BMP U+10FFFD)
+// byte-swapped BOM (non-character U+FFFE)
+varsample='z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE';
+
+test(function(){
+varencoding='utf-8';
+varstring=sample;
+varbytes=[0x7A,0xC2,0xA2,0xE6,0xB0,0xB4,0xF0,0x9D,0x84,0x9E,0xEF,0xA3,0xBF,0xF4,0x8F,0xBF,0xBD,0xEF,0xBF,0xBE];
+varencoded=newTextEncoder().encode(string);
+assert_array_equals([].slice.call(encoded),bytes);
+assert_equals(newTextDecoder(encoding).decode(newUint8Array(bytes)),string);
+assert_equals(newTextDecoder(encoding).decode(newUint8Array(bytes).buffer),string);
+},'Encode/decode round trip: utf-8');
+
+testDecodeSample(
+'utf-16le',
+sample,
+[0x7A,0x00,0xA2,0x00,0x34,0x6C,0x34,0xD8,0x1E,0xDD,0xFF,0xF8,0xFF,0xDB,0xFD,0xDF,0xFE,0xFF]
+);
+
+testDecodeSample(
+'utf-16be',
+sample,
+[0x00,0x7A,0x00,0xA2,0x6C,0x34,0xD8,0x34,0xDD,0x1E,0xF8,0xFF,0xDB,0xFF,0xDF,0xFD,0xFF,0xFE]
+);
+
+testDecodeSample(
+'utf-16',
+sample,
+[0x7A,0x00,0xA2,0x00,0x34,0x6C,0x34,0xD8,0x1E,0xDD,0xFF,0xF8,0xFF,0xDB,0xFD,0xDF,0xFE,0xFF]
+);

test/fixtures/wpt/encoding/api-invalid-label.any.js

@@ -0,0 +1,24 @@
+// META: title=Encoding API: invalid label
+// META: timeout=long
+// META: script=resources/encodings.js
+
+vartests=["invalid-invalidLabel"];
+setup(function(){
+encodings_table.forEach(function(section){
+section.encodings.forEach(function(encoding){
+encoding.labels.forEach(function(label){
+["\u0000","\u000b","\u00a0","\u2028","\u2029"].forEach(function(ws){
+tests.push(ws+label);
+tests.push(label+ws);
+tests.push(ws+label+ws);
+});
+});
+});
+});
+});
+
+tests.forEach(function(input){
+test(function(){
+assert_throws(newRangeError(),function(){newTextDecoder(input);});
+},'Invalid label '+format_value(input)+' should be rejected by TextDecoder.');
+});

test/fixtures/wpt/encoding/api-replacement-encodings.any.js

@@ -0,0 +1,14 @@
+// META: title=Encoding API: replacement encoding
+// META: script=resources/encodings.js
+
+encodings_table.forEach(function(section){
+section.encodings.filter(function(encoding){
+returnencoding.name==='replacement';
+}).forEach(function(encoding){
+encoding.labels.forEach(function(label){
+test(function(){
+assert_throws(newRangeError(),function(){newTextDecoder(label);});
+},'Label for "replacement" should be rejected by API: '+label);
+});
+});
+});

test/fixtures/wpt/encoding/api-surrogates-utf8.any.js

@@ -0,0 +1,48 @@
+// META: title=Encoding API: Invalid UTF-16 surrogates with UTF-8 encoding
+
+varbadStrings=[
+{
+input: 'abc123',
+expected: [0x61,0x62,0x63,0x31,0x32,0x33],
+decoded: 'abc123',
+name: 'Sanity check'
+},
+{
+input: '\uD800',
+expected: [0xef,0xbf,0xbd],
+decoded: '\uFFFD',
+name: 'Surrogate half (low)'
+},
+{
+input: '\uDC00',
+expected: [0xef,0xbf,0xbd],
+decoded: '\uFFFD',
+name: 'Surrogate half (high)'
+},
+{
+input: 'abc\uD800123',
+expected: [0x61,0x62,0x63,0xef,0xbf,0xbd,0x31,0x32,0x33],
+decoded: 'abc\uFFFD123',
+name: 'Surrogate half (low), in a string'
+},
+{
+input: 'abc\uDC00123',
+expected: [0x61,0x62,0x63,0xef,0xbf,0xbd,0x31,0x32,0x33],
+decoded: 'abc\uFFFD123',
+name: 'Surrogate half (high), in a string'
+},
+{
+input: '\uDC00\uD800',
+expected: [0xef,0xbf,0xbd,0xef,0xbf,0xbd],
+decoded: '\uFFFD\uFFFD',
+name: 'Wrong order'
+}
+];
+
+badStrings.forEach(function(t){
+test(function(){
+varencoded=newTextEncoder().encode(t.input);
+assert_array_equals([].slice.call(encoded),t.expected);
+assert_equals(newTextDecoder('utf-8').decode(encoded),t.decoded);
+},'Invalid surrogates encoded into UTF-8: '+t.name);
+});

test/fixtures/wpt/encoding/big5-encoder.html

@@ -0,0 +1,33 @@
+<!doctype html>
+<metacharset=big5><!-- test breaks if the server overrides this -->
+<scriptsrc=/resources/testharness.js></script>
+<scriptsrc=/resources/testharnessreport.js></script>
+<divid=log></div>
+<script>
+functionencode(input,output,desc){
+test(function(){
+vara=document.createElement("a");// <a> uses document encoding for URL's query
+// Append and prepend X to test for off-by-one errors
+a.href="https://example.com/?X"+input+"X";
+assert_equals(a.search.substr(1),"X"+output+"X");// remove leading "?"
+},"big5 encoder: "+desc);
+}
+
+encode("ab","ab","very basic")
+// edge cases
+encode("\u9EA6","%26%2340614%3B","Highest-pointer BMP character excluded from encoder");
+encode("\uD858\uDE6B","%26%23156267%3B","Highest-pointer character excluded from encoder");
+encode("\u3000","%A1@","Lowest-pointer character included in encoder");
+encode("\u20AC","%A3%E1","Euro; the highest-pointer character before a range of 30 unmapped pointers");
+encode("\u4E00","%A4@","The lowest-pointer character after the range of 30 unmapped pointers");
+encode("\uD85D\uDE07","%C8%A4","The highest-pointer character before a range of 41 unmapped pointers");
+encode("\uFFE2","%C8%CD","The lowest-pointer character after the range of 41 unmapped pointers");
+encode("\u79D4","%FE%FE","The last character in the index");
+// not in index
+encode("\u2603","%26%239731%3B","The canonical BMP test character that is not in the index");
+encode("\uD83D\uDCA9","%26%23128169%3B","The canonical astral test character that is not in the index");
+// duplicate low bits
+encode("\uD840\uDFB5","%FDj","A Plane 2 character whose low 16 bits match a BMP character that has a lower pointer");
+// prefer last
+encode("\u2550","%F9%F9","A duplicate-mapped code point that prefers the highest pointer in the encoder");
+</script>

test/fixtures/wpt/encoding/eof-shift_jis-ref.html

@@ -0,0 +1,4 @@
+<!doctype html>
+<metacharset=shift_jis>
+<title>Shift_JIS file ending with a truncated sequence</title>
+One-byte truncated sequence:&#xFFFD;

test/fixtures/wpt/encoding/eof-shift_jis.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<metacharset=shift_jis>
+<title>Shift_JIS file ending with a truncated sequence</title>
+<linkrel=matchhref=/encoding/eof-shift_jis-ref.html>
+One-byte truncated sequence:�

test/fixtures/wpt/encoding/eof-utf-8-one-ref.html

@@ -0,0 +1,4 @@
+<!doctype html>
+<metacharset=utf-8>
+<title>UTF-8 file ending with a one-byte truncated sequence</title>
+One-byte truncated sequence:&#xFFFD;