diff --git a/bin/build-site.sh b/bin/build-site.sh
index eca15dd63..0dff6d0b1 100755
--- a/bin/build-site.sh
+++ b/bin/build-site.sh
@@ -233,6 +233,15 @@ node $TW5_BUILD_TIDDLYWIKI \
--build index \
|| exit 1
+# /editions/twitter-archivist/index.html Twitter Archivist edition
+node $TW5_BUILD_TIDDLYWIKI \
+ ./editions/twitter-archivist \
+ --verbose \
+ --load $TW5_BUILD_OUTPUT/build.tid \
+ --output $TW5_BUILD_OUTPUT/editions/twitter-archivist/ \
+ --build index \
+ || exit 1
+
######################################################
#
# Plugin demos
diff --git a/editions/twitter-archivist/tiddlers/DefaultTiddlers.tid b/editions/twitter-archivist/tiddlers/DefaultTiddlers.tid
new file mode 100644
index 000000000..a3c362aff
--- /dev/null
+++ b/editions/twitter-archivist/tiddlers/DefaultTiddlers.tid
@@ -0,0 +1,3 @@
+title: $:/DefaultTiddlers
+
+HelloThere
\ No newline at end of file
diff --git a/editions/twitter-archivist/tiddlers/HelloThere.tid b/editions/twitter-archivist/tiddlers/HelloThere.tid
new file mode 100644
index 000000000..d1bb60ca7
--- /dev/null
+++ b/editions/twitter-archivist/tiddlers/HelloThere.tid
@@ -0,0 +1,3 @@
+title: HelloThere
+
+{{$:/plugins/tiddlywiki/twitter-archivist/readme}}
diff --git a/editions/twitter-archivist/tiddlers/SiteSubtitle.tid b/editions/twitter-archivist/tiddlers/SiteSubtitle.tid
new file mode 100644
index 000000000..d6b1cd1d4
--- /dev/null
+++ b/editions/twitter-archivist/tiddlers/SiteSubtitle.tid
@@ -0,0 +1,3 @@
+title: $:/SiteTitle
+
+Get Your Tweets Into ~TiddlyWiki
\ No newline at end of file
diff --git a/editions/twitter-archivist/tiddlers/SiteTitle.tid b/editions/twitter-archivist/tiddlers/SiteTitle.tid
new file mode 100644
index 000000000..910c8747d
--- /dev/null
+++ b/editions/twitter-archivist/tiddlers/SiteTitle.tid
@@ -0,0 +1,3 @@
+title: $:/SiteTitle
+
+Twitter Archivist
\ No newline at end of file
diff --git a/editions/twitter-archivist/tiddlywiki.info b/editions/twitter-archivist/tiddlywiki.info
new file mode 100644
index 000000000..681467706
--- /dev/null
+++ b/editions/twitter-archivist/tiddlywiki.info
@@ -0,0 +1,16 @@
+{
+ "description": "Twitter Archivist Edition",
+ "plugins": [
+ "tiddlywiki/twitter-archivist"
+ ],
+ "languages": [
+ ],
+ "themes": [
+ "tiddlywiki/vanilla",
+ "tiddlywiki/snowwhite"
+ ],
+ "build": {
+ "index": [
+ "--rendertiddler","$:/core/save/all","index.html","text/plain"]
+ }
+}
\ No newline at end of file
diff --git a/plugins/tiddlywiki/twitter-archivist/archivist.js b/plugins/tiddlywiki/twitter-archivist/archivist.js
new file mode 100644
index 000000000..2162ee556
--- /dev/null
+++ b/plugins/tiddlywiki/twitter-archivist/archivist.js
@@ -0,0 +1,264 @@
+/*\
+title: $:/plugins/tiddlywiki/twitter-archivist/archivist.js
+type: application/javascript
+module-type: utils
+
+Utility class for manipulating Twitter archives
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+function TwitterArchivist(options) {
+ options = options || {};
+ this.source = options.source;
+}
+
+TwitterArchivist.prototype.loadArchive = async function(options) {
+ options = options || {};
+ const wiki = options.wiki;
+ await this.source.init();
+ // Process the manifest and profile
+ const manifestData = await this.loadTwitterJsData("data/manifest.js","window.__THAR_CONFIG = ",""),
+ profileData = await this.loadTwitterJsData("data/profile.js","window.YTD.profile.part0 = ",""),
+ accountData = await this.loadTwitterJsData("data/account.js","window.YTD.account.part0 = ",""),
+ username = manifestData.userInfo.userName,
+ user_id = manifestData.userInfo.accountId;
+ wiki.addTiddler({
+ title: "Twitter Archive for @" + username,
+ tags: "$:/tags/TwitterArchive",
+ user_id: user_id,
+ username: username,
+ displayname: manifestData.userInfo.displayName,
+ generation_date: $tw.utils.stringifyDate(new Date(manifestData.archiveInfo.generationDate)),
+ account_created_date: $tw.utils.stringifyDate(new Date(accountData[0].account.createdAt)),
+ bio: profileData[0].profile.description.bio,
+ website: profileData[0].profile.description.website,
+ location: profileData[0].profile.description.location
+ });
+ // Process the media
+ await this.source.processFiles("data/tweets_media","base64",function(mediaItem) {
+ var ext = mediaItem.filename.split(".").slice(-1)[0];
+ if("jpg png".split(" ").indexOf(ext) !== -1) {
+ var extensionInfo = $tw.utils.getFileExtensionInfo("." + ext),
+ type = extensionInfo ? extensionInfo.type : null;
+ wiki.addTiddler({
+ title: "Tweet Media - " + mediaItem.filename,
+ tags: "$:/tags/TweetMedia",
+ status_id: mediaItem.filename.split("-")[0],
+ text: mediaItem.contents,
+ type: type
+ });
+ }
+ });
+ // Process the favourites
+ const likeData = await this.loadTwitterJsData("data/like.js","window.YTD.like.part0 = ","");
+ $tw.utils.each(likeData,function(like) {
+ // Create the tweet tiddler
+ var tiddler = {
+ title: "Tweet - " + like.like.tweetId,
+ text: "\\rules only html entity extlink\n" + (like.like.fullText || "").replace("\n","
"),
+ status_id: like.like.tweetId,
+ liked_by: user_id,
+ tags: "$:/tags/Tweet"
+ };
+ wiki.addTiddler(tiddler);
+ });
+ // Process the tweets
+ const tweetData = await this.loadTwitterJsData("data/tweets.js","window.YTD.tweets.part0 = ","");
+ $tw.utils.each(tweetData,function(tweet) {
+ // Compile the tags for the tweet
+ var tags = ["$:/tags/Tweet"];
+ // Create tiddlers for each mentioned tweeter, and hyperlink the mention in the test
+ var rawText = tweet.tweet.full_text,
+ posText = 0,
+ text = "";
+ var mentions = [];
+ $tw.utils.each(tweet.tweet.entities.user_mentions,function(mention) {
+ var title = "Tweeter - " + mention.id_str;
+ tags.push(title);
+ wiki.addTiddler({
+ title: title,
+ screenname: "@" + mention.screen_name,
+ tags: "$:/tags/Tweeter",
+ user_id: mention.id_str,
+ name: mention.name
+ });
+ text = text +
+ $tw.utils.htmlEncode(rawText.substring(posText,mention.indices[0])) +
+ "<$link to=\"" + title + "\">" +
+ $tw.utils.htmlEncode(rawText.substring(mention.indices[0],mention.indices[1])) +
+ "$link>";
+ posText = mention.indices[1];
+ mentions.push(mention.id_str);
+ });
+ if(posText < rawText.length) {
+ text = text + $tw.utils.htmlEncode(rawText.substring(posText));
+ }
+ text = text.replace("\n","
");
+ // Create the tweet tiddler
+ var tiddler = {
+ title: "Tweet - " + tweet.tweet.id_str,
+ text: "\\rules only html entity extlink\n" + text,
+ status_id: tweet.tweet.id_str,
+ user_id: user_id,
+ favorite_count: tweet.tweet.favorite_count,
+ retweet_count: tweet.tweet.retweet_count,
+ tags: tags,
+ created: $tw.utils.stringifyDate(new Date(tweet.tweet.created_at)),
+ modified: $tw.utils.stringifyDate(new Date(tweet.tweet.created_at))
+ };
+ if(tweet.tweet.in_reply_to_status_id_str) {
+ tiddler.in_reply_to_status_id = tweet.tweet.in_reply_to_status_id_str;
+ }
+ if(mentions.length > 0) {
+ tiddler.mention_user_ids = $tw.utils.stringifyList(mentions);
+ }
+ wiki.addTiddler(tiddler);
+ });
+};
+
+TwitterArchivist.prototype.loadTwitterJsData = async function(filePath,prefix,suffix) {
+ var tweetFileData = await this.source.loadTwitterJsData(filePath);
+ if(prefix) {
+ if(tweetFileData.slice(0,prefix.length) !== prefix) {
+ throw "Reading Twitter JS file " + filePath + " missing prefix '" + prefix + "'";
+ }
+ tweetFileData = tweetFileData.slice(prefix.length);
+ }
+ if(suffix) {
+ if(tweetFileData.slice(-suffix.length) !== suffix) {
+ throw "Reading Twitter JS file " + filePath + " missing suffix '" + suffix + "'";
+ }
+ tweetFileData = tweetFileData.slice(0,tweetFileData.length - suffix.length);
+ }
+ return JSON.parse(tweetFileData);
+};
+
+function TwitterArchivistSourceNodeJs(options) {
+ options = options || {};
+ this.archivePath = options.archivePath;
+}
+
+TwitterArchivistSourceNodeJs.prototype.init = async function() {
+};
+
+TwitterArchivistSourceNodeJs.prototype.processFiles = async function(dirPath,encoding,callback) {
+ var fs = require("fs"),
+ path = require("path"),
+ dirPath = path.resolve(this.archivePath,dirPath),
+ filenames = fs.readdirSync(dirPath);
+ $tw.utils.each(filenames,function(filename) {
+ callback({
+ filename: filename,
+ contents: fs.readFileSync(path.resolve(dirPath,filename),encoding)
+ });
+ });
+};
+
+TwitterArchivistSourceNodeJs.prototype.loadTwitterJsData = async function(filePath) {
+ var fs = require("fs"),
+ path = require("path");
+ return fs.readFileSync(path.resolve(this.archivePath,filePath),"utf8");
+};
+
+function TwitterArchivistSourceBrowser(options) {
+ options = options || {};
+}
+
+TwitterArchivistSourceBrowser.prototype.init = async function() {
+ // Open directory
+ this.rootDirHandle = await window.showDirectoryPicker();
+};
+
+TwitterArchivistSourceBrowser.prototype.processFiles = async function(dirPath,encoding,callback) {
+ const dirHandle = await this.walkDirectory(dirPath.split("/"));
+ for await (const [filename, fileHandle] of dirHandle.entries()) {
+ const contents = await fileHandle.getFile();
+ callback({
+ filename: filename,
+ contents: arrayBufferToBase64(await contents.arrayBuffer())
+ });
+ }
+};
+
+TwitterArchivistSourceBrowser.prototype.loadTwitterJsData = async function(filePath) {
+ const filePathParts = filePath.split("/");
+ const dirHandle = await this.walkDirectory(filePathParts.slice(0,-1));
+ const fileHandle = await dirHandle.getFileHandle(filePathParts.slice(-1)[0]);
+ const contents = await fileHandle.getFile();
+ return await contents.text();
+};
+
+TwitterArchivistSourceBrowser.prototype.walkDirectory = async function(arrayDirectoryEntries) {
+ var entries = arrayDirectoryEntries.slice(0),
+ dirHandle = this.rootDirHandle;
+ while(entries.length > 0) {
+ dirHandle = await dirHandle.getDirectoryHandle(entries[0]);
+ entries.shift();
+ }
+ return dirHandle;
+};
+
+// Thanks to MatheusFelipeMarinho
+// https://github.com/MatheusFelipeMarinho/venom/blob/43ead0bfffa57a536a5cff67dd909e55da9f0915/src/lib/wapi/helper/array-buffer-to-base64.js#L55
+function arrayBufferToBase64(arrayBuffer) {
+ var base64 = '';
+ var encodings =
+ 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+
+ var bytes = new Uint8Array(arrayBuffer);
+ var byteLength = bytes.byteLength;
+ var byteRemainder = byteLength % 3;
+ var mainLength = byteLength - byteRemainder;
+
+ var a, b, c, d;
+ var chunk;
+
+ // Main loop deals with bytes in chunks of 3
+ for (var i = 0; i < mainLength; i = i + 3) {
+ // Combine the three bytes into a single integer
+ chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
+
+ // Use bitmasks to extract 6-bit segments from the triplet
+ a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
+ b = (chunk & 258048) >> 12; // 258048 = (2^6 - 1) << 12
+ c = (chunk & 4032) >> 6; // 4032 = (2^6 - 1) << 6
+ d = chunk & 63; // 63 = 2^6 - 1
+
+ // Convert the raw binary segments to the appropriate ASCII encoding
+ base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
+ }
+
+ // Deal with the remaining bytes and padding
+ if (byteRemainder == 1) {
+ chunk = bytes[mainLength];
+
+ a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2
+
+ // Set the 4 least significant bits to zero
+ b = (chunk & 3) << 4; // 3 = 2^2 - 1
+
+ base64 += encodings[a] + encodings[b] + '==';
+ } else if (byteRemainder == 2) {
+ chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];
+
+ a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
+ b = (chunk & 1008) >> 4; // 1008 = (2^6 - 1) << 4
+
+ // Set the 2 least significant bits to zero
+ c = (chunk & 15) << 2; // 15 = 2^4 - 1
+
+ base64 += encodings[a] + encodings[b] + encodings[c] + '=';
+ }
+ return base64;
+}
+
+exports.TwitterArchivist = TwitterArchivist;
+exports.TwitterArchivistSourceNodeJs = TwitterArchivistSourceNodeJs;
+exports.TwitterArchivistSourceBrowser = TwitterArchivistSourceBrowser;
+
+})();
diff --git a/plugins/tiddlywiki/twitter-archivist/configTiddlerInfoMode.tid b/plugins/tiddlywiki/twitter-archivist/configTiddlerInfoMode.tid
new file mode 100644
index 000000000..3b3716299
--- /dev/null
+++ b/plugins/tiddlywiki/twitter-archivist/configTiddlerInfoMode.tid
@@ -0,0 +1,2 @@
+title: $:/config/TiddlerInfo/Mode
+text: sticky
diff --git a/plugins/tiddlywiki/twitter-archivist/loadtwitterarchive.js b/plugins/tiddlywiki/twitter-archivist/loadtwitterarchive.js
new file mode 100644
index 000000000..497b82bf5
--- /dev/null
+++ b/plugins/tiddlywiki/twitter-archivist/loadtwitterarchive.js
@@ -0,0 +1,53 @@
+/*\
+title: $:/plugins/tiddlywiki/twitter-archivist/loadtwitterarchive.js
+type: application/javascript
+module-type: command
+
+Read tiddlers from an unzipped Twitter archive
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+var widget = require("$:/core/modules/widgets/widget.js");
+
+exports.info = {
+ name: "loadtwitterarchive",
+ synchronous: false
+};
+
+var Command = function(params,commander,callback) {
+ this.params = params;
+ this.commander = commander;
+ this.callback = callback;
+};
+
+Command.prototype.execute = function() {
+ var self = this;
+ if(this.params.length < 1) {
+ return "Missing path to Twitter archive";
+ }
+ var archivePath = this.params[0];
+ // Load tweets
+ var archiveSource = new $tw.utils.TwitterArchivistSourceNodeJs({
+ archivePath: archivePath
+ }),
+ archivist = new $tw.utils.TwitterArchivist({
+ source: archiveSource
+ });
+ archivist.loadArchive({
+ wiki: this.commander.wiki
+ }).then(function() {
+ self.callback(null);
+ }).catch(function(err) {
+ self.callback(err);
+ });
+ return null;
+};
+
+exports.Command = Command;
+
+})();
diff --git a/plugins/tiddlywiki/twitter-archivist/macros.tid b/plugins/tiddlywiki/twitter-archivist/macros.tid
new file mode 100644
index 000000000..c0003274d
--- /dev/null
+++ b/plugins/tiddlywiki/twitter-archivist/macros.tid
@@ -0,0 +1,204 @@
+title: $:/plugins/tiddlywiki/twitter-archivist/macros
+tags: $:/tags/Macro
+
+\define skinny-tabs(tabNames,tabCaptions,defaultTab,state)
+<$let
+ currTab={{{ [<__state__>get[text]else<__defaultTab__>] }}}
+>
+
Username | <$text text={{!!screenname}}/> |
---|---|
Display Name | <$text text={{!!name}}/> |
User ID | <$text text={{!!user_id}}/> |