From cae3d0fa2c9f9d8a5dfbcb9a38ba5403f0a6ec1f Mon Sep 17 00:00:00 2001 From: Leilei332 Date: Mon, 6 Oct 2025 23:25:45 +0800 Subject: [PATCH] Switch to native support for base64 utf8 (#9253) * Switch to native support for base64 utf8 * Fix error on nodejs --- .../utils/base64-utf8/base64-utf8.module.js | 142 ------------------ .../base64-utf8/base64-utf8.module.min.js | 9 -- .../utils/base64-utf8/tiddlywiki.files | 14 -- core/modules/utils/utils.js | 38 ++++- 4 files changed, 32 insertions(+), 171 deletions(-) delete mode 100644 core/modules/utils/base64-utf8/base64-utf8.module.js delete mode 100644 core/modules/utils/base64-utf8/base64-utf8.module.min.js delete mode 100644 core/modules/utils/base64-utf8/tiddlywiki.files diff --git a/core/modules/utils/base64-utf8/base64-utf8.module.js b/core/modules/utils/base64-utf8/base64-utf8.module.js deleted file mode 100644 index 8bd4e272d..000000000 --- a/core/modules/utils/base64-utf8/base64-utf8.module.js +++ /dev/null @@ -1,142 +0,0 @@ -// From https://gist.github.com/Nijikokun/5192472 -// -// UTF8 Module -// -// Cleaner and modularized utf-8 encoding and decoding library for javascript. -// -// copyright: MIT -// author: Nijiko Yonskai, @nijikokun, nijikokun@gmail.com -(function (name, definition, context, dependencies) { - if (typeof context['module'] !== 'undefined' && context['module']['exports']) { if (dependencies && context['require']) { for (var i = 0; i < dependencies.length; i++) context[dependencies[i]] = context['require'](dependencies[i]); } context['module']['exports'] = definition.apply(context); } - else if (typeof context['define'] !== 'undefined' && context['define'] === 'function' && context['define']['amd']) { define(name, (dependencies || []), definition); } - else { context[name] = definition.apply(context); } -})('utf8', function () { - return { - encode: function (string) { - if (typeof string !== 'string') return string; - else string = string.replace(/\r\n/g, "\n"); - var output = "", i = 0, charCode; - - for (i; i < string.length; i++) { - charCode = string.charCodeAt(i); - - if (charCode < 128) { - output += String.fromCharCode(charCode); - } else if ((charCode > 127) && (charCode < 2048)) { - output += String.fromCharCode((charCode >> 6) | 192); - output += String.fromCharCode((charCode & 63) | 128); - } else if ((charCode > 55295) && (charCode < 57344) && string.length > i+1) { - // Surrogate pair - var hiSurrogate = charCode; - var loSurrogate = string.charCodeAt(i+1); - i++; // Skip the low surrogate on the next loop pass - var codePoint = (((hiSurrogate - 55296) << 10) | (loSurrogate - 56320)) + 65536; - output += String.fromCharCode((codePoint >> 18) | 240); - output += String.fromCharCode(((codePoint >> 12) & 63) | 128); - output += String.fromCharCode(((codePoint >> 6) & 63) | 128); - output += String.fromCharCode((codePoint & 63) | 128); - } else { - // Not a surrogate pair, or a dangling surrogate without its partner that we'll just encode as-is - output += String.fromCharCode((charCode >> 12) | 224); - output += String.fromCharCode(((charCode >> 6) & 63) | 128); - output += String.fromCharCode((charCode & 63) | 128); - } - } - - return output; - }, - - decode: function (string) { - if (typeof string !== 'string') return string; - var output = "", i = 0, charCode = 0; - - while (i < string.length) { - charCode = string.charCodeAt(i); - - if (charCode < 128) { - output += String.fromCharCode(charCode), - i++; - } else if ((charCode > 191) && (charCode < 224)) { - output += String.fromCharCode(((charCode & 31) << 6) | (string.charCodeAt(i + 1) & 63)); - i += 2; - } else if ((charCode > 223) && (charCode < 240)) { - output += String.fromCharCode(((charCode & 15) << 12) | ((string.charCodeAt(i + 1) & 63) << 6) | (string.charCodeAt(i + 2) & 63)); - i += 3; - } else { - var codePoint = ((charCode & 7) << 18) | ((string.charCodeAt(i + 1) & 63) << 12) | ((string.charCodeAt(i + 2) & 63) << 6) | (string.charCodeAt(i + 3) & 63); - // output += String.fromCodePoint(codePoint); // Can't do this because Internet Explorer doesn't have String.fromCodePoint - output += String.fromCharCode(((codePoint - 65536) >> 10) + 55296) + String.fromCharCode(((codePoint - 65536) & 1023) + 56320); // So we do this instead - i += 4; - } - } - - return output; - } - }; -}, this); - -// Base64 Module -// -// Cleaner, modularized and properly scoped base64 encoding and decoding module for strings. -// -// copyright: MIT -// author: Nijiko Yonskai, @nijikokun, nijikokun@gmail.com -(function (name, definition, context, dependencies) { - if (typeof context['module'] !== 'undefined' && context['module']['exports']) { if (dependencies && context['require']) { for (var i = 0; i < dependencies.length; i++) context[dependencies[i]] = context['require'](dependencies[i]); } context['module']['exports'] = definition.apply(context); } - else if (typeof context['define'] !== 'undefined' && context['define'] === 'function' && context['define']['amd']) { define(name, (dependencies || []), definition); } - else { context[name] = definition.apply(context); } -})('base64', function (utf8) { - var $this = this; - var $utf8 = utf8 || this.utf8; - var map = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; - - return { - encode: function (input) { - if (typeof $utf8 === 'undefined') throw { error: "MissingMethod", message: "UTF8 Module is missing." }; - if (typeof input !== 'string') return input; - else input = $utf8.encode(input); - var output = "", a, b, c, d, e, f, g, i = 0; - - while (i < input.length) { - a = input.charCodeAt(i++); - b = input.charCodeAt(i++); - c = input.charCodeAt(i++); - d = a >> 2; - e = ((a & 3) << 4) | (b >> 4); - f = ((b & 15) << 2) | (c >> 6); - g = c & 63; - - if (isNaN(b)) f = g = 64; - else if (isNaN(c)) g = 64; - - output += map.charAt(d) + map.charAt(e) + map.charAt(f) + map.charAt(g); - } - - return output; - }, - - decode: function (input) { - if (typeof $utf8 === 'undefined') throw { error: "MissingMethod", message: "UTF8 Module is missing." }; - if (typeof input !== 'string') return input; - else input = input.replace(/[^A-Za-z0-9\+\/\=]/g, ""); - var output = "", a, b, c, d, e, f, g, i = 0; - - while (i < input.length) { - d = map.indexOf(input.charAt(i++)); - e = map.indexOf(input.charAt(i++)); - f = map.indexOf(input.charAt(i++)); - g = map.indexOf(input.charAt(i++)); - - a = (d << 2) | (e >> 4); - b = ((e & 15) << 4) | (f >> 2); - c = ((f & 3) << 6) | g; - - output += String.fromCharCode(a); - if (f != 64) output += String.fromCharCode(b); - if (g != 64) output += String.fromCharCode(c); - } - - return $utf8.decode(output); - } - } -}, this, [ "utf8" ]); \ No newline at end of file diff --git a/core/modules/utils/base64-utf8/base64-utf8.module.min.js b/core/modules/utils/base64-utf8/base64-utf8.module.min.js deleted file mode 100644 index f2f808004..000000000 --- a/core/modules/utils/base64-utf8/base64-utf8.module.min.js +++ /dev/null @@ -1,9 +0,0 @@ -// From https://gist.github.com/Nijikokun/5192472 -// -// UTF8 Module -// -// Cleaner and modularized utf-8 encoding and decoding library for javascript. -// -// copyright: MIT -// author: Nijiko Yonskai, @nijikokun, nijikokun@gmail.com -!function(r,e,o,t){void 0!==o.module&&o.module.exports?o.module.exports=e.apply(o):void 0!==o.define&&"function"===o.define&&o.define.amd?define("utf8",[],e):o.utf8=e.apply(o)}(0,function(){return{encode:function(r){if("string"!=typeof r)return r;r=r.replace(/\r\n/g,"\n");for(var e,o="",t=0;t127&&e<2048)o+=String.fromCharCode(e>>6|192),o+=String.fromCharCode(63&e|128);else if(e>55295&&e<57344&&r.length>t+1){var i=e,n=r.charCodeAt(t+1);t++;var d=65536+(i-55296<<10|n-56320);o+=String.fromCharCode(d>>18|240),o+=String.fromCharCode(d>>12&63|128),o+=String.fromCharCode(d>>6&63|128),o+=String.fromCharCode(63&d|128)}else o+=String.fromCharCode(e>>12|224),o+=String.fromCharCode(e>>6&63|128),o+=String.fromCharCode(63&e|128);return o},decode:function(r){if("string"!=typeof r)return r;for(var e="",o=0,t=0;o191&&t<224)e+=String.fromCharCode((31&t)<<6|63&r.charCodeAt(o+1)),o+=2;else if(t>223&&t<240)e+=String.fromCharCode((15&t)<<12|(63&r.charCodeAt(o+1))<<6|63&r.charCodeAt(o+2)),o+=3;else{var i=(7&t)<<18|(63&r.charCodeAt(o+1))<<12|(63&r.charCodeAt(o+2))<<6|63&r.charCodeAt(o+3);e+=String.fromCharCode(55296+(i-65536>>10))+String.fromCharCode(56320+(i-65536&1023)),o+=4}return e}}},this),function(r,e,o,t){if(void 0!==o.module&&o.module.exports){if(t&&o.require)for(var i=0;i>2,f=(3&t)<<4|(i=r.charCodeAt(c++))>>4,a=(15&i)<<2|(n=r.charCodeAt(c++))>>6,h=63&n,isNaN(i)?a=h=64:isNaN(n)&&(h=64),C+=o.charAt(d)+o.charAt(f)+o.charAt(a)+o.charAt(h);return C},decode:function(r){if(void 0===e)throw{error:"MissingMethod",message:"UTF8 Module is missing."};if("string"!=typeof r)return r;r=r.replace(/[^A-Za-z0-9\+\/\=]/g,"");for(var t,i,n,d,f,a,h="",C=0;C>4,i=(15&d)<<4|(f=o.indexOf(r.charAt(C++)))>>2,n=(3&f)<<6|(a=o.indexOf(r.charAt(C++))),h+=String.fromCharCode(t),64!=f&&(h+=String.fromCharCode(i)),64!=a&&(h+=String.fromCharCode(n));return e.decode(h)}}},this,["utf8"]); \ No newline at end of file diff --git a/core/modules/utils/base64-utf8/tiddlywiki.files b/core/modules/utils/base64-utf8/tiddlywiki.files deleted file mode 100644 index b12e7dfb9..000000000 --- a/core/modules/utils/base64-utf8/tiddlywiki.files +++ /dev/null @@ -1,14 +0,0 @@ -{ - "tiddlers": [ - { - "file": "base64-utf8.module.min.js", - "fields": { - "type": "application/javascript", - "title": "$:/core/modules/utils/base64-utf8/base64-utf8.module.js", - "module-type": "library" - }, - "prefix": "(function(){", - "suffix": "}).call(exports);" - } - ] -} diff --git a/core/modules/utils/utils.js b/core/modules/utils/utils.js index a228f91d4..93abf0006 100644 --- a/core/modules/utils/utils.js +++ b/core/modules/utils/utils.js @@ -9,8 +9,6 @@ Various static utility functions. "use strict"; -var base64utf8 = require("$:/core/modules/utils/base64-utf8/base64-utf8.module.js"); - /* Display a message, in colour if we're on a terminal */ @@ -842,22 +840,50 @@ if(typeof window !== 'undefined') { } } +exports.base64ToBytes = function(base64) { + const binString = exports.atob(base64); + return Uint8Array.from(binString, (m) => m.codePointAt(0)); +}; + +exports.bytesToBase64 = function(bytes) { + const binString = Array.from(bytes, (byte) => String.fromCodePoint(byte)).join(""); + return exports.btoa(binString); +}; + +exports.base64EncodeUtf8 = function(str) { + if ($tw.browser) { + return exports.bytesToBase64(new TextEncoder().encode(str)); + } else { + const buff = Buffer.from(str, "utf-8"); + return buff.toString("base64"); + } +}; + +exports.base64DecodeUtf8 = function(str) { + if ($tw.browser) { + return new TextDecoder().decode(exports.base64ToBytes(str)); + } else { + const buff = Buffer.from(str, "base64"); + return buff.toString("utf-8"); + } +}; + /* Decode a base64 string */ exports.base64Decode = function(string64,binary,urlsafe) { - var encoded = urlsafe ? string64.replace(/_/g,'/').replace(/-/g,'+') : string64; + const encoded = urlsafe ? string64.replace(/_/g,'/').replace(/-/g,'+') : string64; if(binary) return exports.atob(encoded) - else return base64utf8.base64.decode.call(base64utf8,encoded); + else return exports.base64DecodeUtf8(encoded); }; /* Encode a string to base64 */ exports.base64Encode = function(string64,binary,urlsafe) { - var encoded; + let encoded; if(binary) encoded = exports.btoa(string64); - else encoded = base64utf8.base64.encode.call(base64utf8,string64); + else encoded = exports.base64EncodeUtf8(string64); if(urlsafe) { encoded = encoded.replace(/\+/g,'-').replace(/\//g,'_'); }