Source: lib/util/string_utils.js

  1. /*! @license
  2. * Shaka Player
  3. * Copyright 2016 Google LLC
  4. * SPDX-License-Identifier: Apache-2.0
  5. */
  6. goog.provide('shaka.util.StringUtils');
  7. goog.require('goog.asserts');
  8. goog.require('shaka.log');
  9. goog.require('shaka.util.BufferUtils');
  10. goog.require('shaka.util.Error');
  11. goog.require('shaka.util.Lazy');
  12. goog.require('shaka.util.Platform');
  13. /**
  14. * @namespace shaka.util.StringUtils
  15. * @summary A set of string utility functions.
  16. * @export
  17. */
  18. shaka.util.StringUtils = class {
  19. /**
  20. * Creates a string from the given buffer as UTF-8 encoding.
  21. *
  22. * @param {?BufferSource} data
  23. * @return {string}
  24. * @export
  25. */
  26. static fromUTF8(data) {
  27. if (!data) {
  28. return '';
  29. }
  30. let uint8 = shaka.util.BufferUtils.toUint8(data);
  31. // If present, strip off the UTF-8 BOM.
  32. if (uint8[0] == 0xef && uint8[1] == 0xbb && uint8[2] == 0xbf) {
  33. uint8 = uint8.subarray(3);
  34. }
  35. if (window.TextDecoder && !shaka.util.Platform.isPS4()) {
  36. // Use the TextDecoder interface to decode the text. This has the
  37. // advantage compared to the previously-standard decodeUriComponent that
  38. // it will continue parsing even if it finds an invalid UTF8 character,
  39. // rather than stop and throw an error.
  40. const utf8decoder = new TextDecoder();
  41. const decoded = utf8decoder.decode(uint8);
  42. if (decoded.includes('\uFFFD')) {
  43. shaka.log.alwaysError('Decoded string contains an "unknown character' +
  44. '" codepoint. That probably means the UTF8 ' +
  45. 'encoding was incorrect!');
  46. }
  47. return decoded;
  48. } else {
  49. // Homebrewed UTF-8 decoder based on
  50. // https://en.wikipedia.org/wiki/UTF-8#Encoding
  51. // Unlike decodeURIComponent, won't throw on bad encoding.
  52. // In this way, it is similar to TextDecoder.
  53. let decoded = '';
  54. for (let i = 0; i < uint8.length; ++i) {
  55. // By default, the "replacement character" codepoint.
  56. let codePoint = 0xFFFD;
  57. // Top bit is 0, 1-byte encoding.
  58. if ((uint8[i] & 0x80) == 0) {
  59. codePoint = uint8[i];
  60. // Top 3 bits of byte 0 are 110, top 2 bits of byte 1 are 10,
  61. // 2-byte encoding.
  62. } else if (uint8.length >= i + 2 &&
  63. (uint8[i] & 0xe0) == 0xc0 &&
  64. (uint8[i + 1] & 0xc0) == 0x80) {
  65. codePoint = ((uint8[i] & 0x1f) << 6) |
  66. ((uint8[i + 1] & 0x3f));
  67. i += 1; // Consume one extra byte.
  68. // Top 4 bits of byte 0 are 1110, top 2 bits of byte 1 and 2 are 10,
  69. // 3-byte encoding.
  70. } else if (uint8.length >= i + 3 &&
  71. (uint8[i] & 0xf0) == 0xe0 &&
  72. (uint8[i + 1] & 0xc0) == 0x80 &&
  73. (uint8[i + 2] & 0xc0) == 0x80) {
  74. codePoint = ((uint8[i] & 0x0f) << 12) |
  75. ((uint8[i + 1] & 0x3f) << 6) |
  76. ((uint8[i + 2] & 0x3f));
  77. i += 2; // Consume two extra bytes.
  78. // Top 5 bits of byte 0 are 11110, top 2 bits of byte 1, 2 and 3 are 10,
  79. // 4-byte encoding.
  80. } else if (uint8.length >= i + 4 &&
  81. (uint8[i] & 0xf1) == 0xf0 &&
  82. (uint8[i + 1] & 0xc0) == 0x80 &&
  83. (uint8[i + 2] & 0xc0) == 0x80 &&
  84. (uint8[i + 3] & 0xc0) == 0x80) {
  85. codePoint = ((uint8[i] & 0x07) << 18) |
  86. ((uint8[i + 1] & 0x3f) << 12) |
  87. ((uint8[i + 2] & 0x3f) << 6) |
  88. ((uint8[i + 3] & 0x3f));
  89. i += 3; // Consume three extra bytes.
  90. }
  91. // JavaScript strings are a series of UTF-16 characters.
  92. if (codePoint <= 0xffff) {
  93. decoded += String.fromCharCode(codePoint);
  94. } else {
  95. // UTF-16 surrogate-pair encoding, based on
  96. // https://en.wikipedia.org/wiki/UTF-16#Description
  97. const baseCodePoint = codePoint - 0x10000;
  98. const highPart = baseCodePoint >> 10;
  99. const lowPart = baseCodePoint & 0x3ff;
  100. decoded += String.fromCharCode(0xd800 + highPart);
  101. decoded += String.fromCharCode(0xdc00 + lowPart);
  102. }
  103. }
  104. return decoded;
  105. }
  106. }
  107. /**
  108. * Creates a string from the given buffer as UTF-16 encoding.
  109. *
  110. * @param {?BufferSource} data
  111. * @param {boolean} littleEndian
  112. true to read little endian, false to read big.
  113. * @param {boolean=} noThrow true to avoid throwing in cases where we may
  114. * expect invalid input. If noThrow is true and the data has an odd
  115. * length,it will be truncated.
  116. * @return {string}
  117. * @export
  118. */
  119. static fromUTF16(data, littleEndian, noThrow) {
  120. if (!data) {
  121. return '';
  122. }
  123. if (!noThrow && data.byteLength % 2 != 0) {
  124. shaka.log.error('Data has an incorrect length, must be even.');
  125. throw new shaka.util.Error(
  126. shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.TEXT,
  127. shaka.util.Error.Code.BAD_ENCODING);
  128. }
  129. // Use a DataView to ensure correct endianness.
  130. const length = Math.floor(data.byteLength / 2);
  131. const arr = new Uint16Array(length);
  132. const dataView = shaka.util.BufferUtils.toDataView(data);
  133. for (let i = 0; i < length; i++) {
  134. arr[i] = dataView.getUint16(i * 2, littleEndian);
  135. }
  136. return shaka.util.StringUtils.fromCharCode(arr);
  137. }
  138. /**
  139. * Creates a string from the given buffer, auto-detecting the encoding that is
  140. * being used. If it cannot detect the encoding, it will throw an exception.
  141. *
  142. * @param {?BufferSource} data
  143. * @return {string}
  144. * @export
  145. */
  146. static fromBytesAutoDetect(data) {
  147. const StringUtils = shaka.util.StringUtils;
  148. if (!data) {
  149. return '';
  150. }
  151. const uint8 = shaka.util.BufferUtils.toUint8(data);
  152. if (uint8[0] == 0xef && uint8[1] == 0xbb && uint8[2] == 0xbf) {
  153. return StringUtils.fromUTF8(uint8);
  154. } else if (uint8[0] == 0xfe && uint8[1] == 0xff) {
  155. return StringUtils.fromUTF16(
  156. uint8.subarray(2), /* littleEndian= */ false);
  157. } else if (uint8[0] == 0xff && uint8[1] == 0xfe) {
  158. return StringUtils.fromUTF16(uint8.subarray(2), /* littleEndian= */ true);
  159. }
  160. const isAscii = (i) => {
  161. // arr[i] >= ' ' && arr[i] <= '~';
  162. return uint8.byteLength <= i || (uint8[i] >= 0x20 && uint8[i] <= 0x7e);
  163. };
  164. shaka.log.debug(
  165. 'Unable to find byte-order-mark, making an educated guess.');
  166. if (uint8[0] == 0 && uint8[2] == 0) {
  167. return StringUtils.fromUTF16(data, /* littleEndian= */ false);
  168. } else if (uint8[1] == 0 && uint8[3] == 0) {
  169. return StringUtils.fromUTF16(data, /* littleEndian= */ true);
  170. } else if (isAscii(0) && isAscii(1) && isAscii(2) && isAscii(3)) {
  171. return StringUtils.fromUTF8(data);
  172. }
  173. throw new shaka.util.Error(
  174. shaka.util.Error.Severity.CRITICAL,
  175. shaka.util.Error.Category.TEXT,
  176. shaka.util.Error.Code.UNABLE_TO_DETECT_ENCODING);
  177. }
  178. /**
  179. * Creates a ArrayBuffer from the given string, converting to UTF-8 encoding.
  180. *
  181. * @param {string} str
  182. * @return {!ArrayBuffer}
  183. * @export
  184. */
  185. static toUTF8(str) {
  186. if (window.TextEncoder && !shaka.util.Platform.isPS4()) {
  187. const utf8Encoder = new TextEncoder();
  188. return shaka.util.BufferUtils.toArrayBuffer(utf8Encoder.encode(str));
  189. } else {
  190. // http://stackoverflow.com/a/13691499
  191. // Converts the given string to a URI encoded string. If a character
  192. // falls in the ASCII range, it is not converted; otherwise it will be
  193. // converted to a series of URI escape sequences according to UTF-8.
  194. // Example: 'g#€' -> 'g#%E3%82%AC'
  195. const encoded = encodeURIComponent(str);
  196. // Convert each escape sequence individually into a character. Each
  197. // escape sequence is interpreted as a code-point, so if an escape
  198. // sequence happens to be part of a multi-byte sequence, each byte will
  199. // be converted to a single character.
  200. // Example: 'g#%E3%82%AC' -> '\x67\x35\xe3\x82\xac'
  201. const utf8 = unescape(encoded);
  202. const result = new Uint8Array(utf8.length);
  203. for (let i = 0; i < utf8.length; i++) {
  204. const item = utf8[i];
  205. result[i] = item.charCodeAt(0);
  206. }
  207. return shaka.util.BufferUtils.toArrayBuffer(result);
  208. }
  209. }
  210. /**
  211. * Creates a ArrayBuffer from the given string, converting to UTF-16 encoding.
  212. *
  213. * @param {string} str
  214. * @param {boolean} littleEndian
  215. * @return {!ArrayBuffer}
  216. * @export
  217. */
  218. static toUTF16(str, littleEndian) {
  219. const result = new ArrayBuffer(str.length * 2);
  220. const view = new DataView(result);
  221. for (let i = 0; i < str.length; ++i) {
  222. const value = str.charCodeAt(i);
  223. view.setUint16(/* position= */ i * 2, value, littleEndian);
  224. }
  225. return result;
  226. }
  227. /**
  228. * Creates a new string from the given array of char codes.
  229. *
  230. * Using String.fromCharCode.apply is risky because you can trigger stack
  231. * errors on very large arrays. This breaks up the array into several pieces
  232. * to avoid this.
  233. *
  234. * @param {!TypedArray} array
  235. * @return {string}
  236. */
  237. static fromCharCode(array) {
  238. return shaka.util.StringUtils.fromCharCodeImpl_.value()(array);
  239. }
  240. /**
  241. * Resets the fromCharCode method's implementation.
  242. * For debug use.
  243. * @export
  244. */
  245. static resetFromCharCode() {
  246. shaka.util.StringUtils.fromCharCodeImpl_.reset();
  247. }
  248. /**
  249. * This method converts the HTML entities &amp;, &lt;, &gt;, &quot;, &#39;,
  250. * &nbsp;, &lrm; and &rlm; in string to their corresponding characters.
  251. *
  252. * @param {!string} input
  253. * @return {string}
  254. */
  255. static htmlUnescape(input) {
  256. // Used to map HTML entities to characters.
  257. const htmlUnescapes = {
  258. '&amp;': '&',
  259. '&lt;': '<',
  260. '&gt;': '>',
  261. '&quot;': '"',
  262. '&apos;': '\'',
  263. '&nbsp;': '\u{a0}',
  264. '&lrm;': '\u{200e}',
  265. '&rlm;': '\u{200f}',
  266. };
  267. // Used to match HTML entities and HTML characters.
  268. const reEscapedHtml =
  269. /&(?:amp|lt|gt|quot|apos|nbsp|lrm|rlm|#[xX]?[0-9a-fA-F]+);/g;
  270. const reHasEscapedHtml = RegExp(reEscapedHtml.source);
  271. // This check is an optimization, since replace always makes a copy
  272. if (input && reHasEscapedHtml.test(input)) {
  273. return input.replace(reEscapedHtml, (entity) => {
  274. if (entity[1] == '#') {
  275. // Translate this into an HTML character.
  276. let code = 0;
  277. if (entity[2] == 'x' || entity[2] == 'X') {
  278. // It's hex.
  279. code = parseInt(entity.substring(3), 16);
  280. } else {
  281. // It's decimal.
  282. code = parseInt(entity.substring(2), 10);
  283. }
  284. // Ignore it if it's an invalid code point.
  285. if (code >= 0 && code <= 0x10FFFF) {
  286. return String.fromCodePoint(code);
  287. } else {
  288. return '';
  289. }
  290. }
  291. // The only thing that might not match the dictionary above is the
  292. // single quote, which can be matched by many strings in the regex, but
  293. // only has a single entry in the dictionary.
  294. return htmlUnescapes[entity] || '\'';
  295. });
  296. }
  297. return input || '';
  298. }
  299. };
  300. /** @private {!shaka.util.Lazy.<function(!TypedArray):string>} */
  301. shaka.util.StringUtils.fromCharCodeImpl_ = new shaka.util.Lazy(() => {
  302. /**
  303. * @param {number} size
  304. * @return {boolean}
  305. */
  306. const supportsChunkSize = (size) => {
  307. try {
  308. // The compiler will complain about suspicious value if this isn't
  309. // stored in a variable and used.
  310. const buffer = new Uint8Array(size);
  311. // This can't use the spread operator, or it blows up on Xbox One.
  312. // So we use apply() instead, which is normally not allowed.
  313. // See issue #2186 for more details.
  314. const foo = String.fromCharCode.apply(null, buffer);
  315. goog.asserts.assert(foo, 'Should get value');
  316. return foo.length > 0; // Actually use "foo", so it's not compiled out.
  317. } catch (error) {
  318. return false;
  319. }
  320. };
  321. // Different browsers support different chunk sizes; find out the largest
  322. // this browser supports so we can use larger chunks on supported browsers
  323. // but still support lower-end devices that require small chunks.
  324. // 64k is supported on all major desktop browsers.
  325. for (let size = 64 * 1024; size > 0; size /= 2) {
  326. if (supportsChunkSize(size)) {
  327. return (buffer) => {
  328. let ret = '';
  329. for (let i = 0; i < buffer.length; i += size) {
  330. const subArray = buffer.subarray(i, i + size);
  331. // This can't use the spread operator, or it blows up on Xbox One.
  332. // So we use apply() instead, which is normally not allowed.
  333. // See issue #2186 for more details.
  334. ret += String.fromCharCode.apply(null, subArray); // Issue #2186
  335. }
  336. return ret;
  337. };
  338. }
  339. }
  340. goog.asserts.assert(false, 'Unable to create a fromCharCode method');
  341. return null;
  342. });