From af6c0170868c2cd699f57cef814205368a234c51 Mon Sep 17 00:00:00 2001
From: "jeremy@jermolene.com" <jeremy@jermolene.com>
Date: Sun, 13 Nov 2022 19:05:09 +0000
Subject: [PATCH] Twitter Archivist: Expand t.co URLs in tweets

---
 .../tiddlywiki/twitter-archivist/archivist.js | 63 ++++++++++++++-----
 plugins/tiddlywiki/twitter-archivist/spec.tid |  1 +
 plugins/tiddlywiki/twitter-archivist/todo.tid |  3 +-
 3 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/plugins/tiddlywiki/twitter-archivist/archivist.js b/plugins/tiddlywiki/twitter-archivist/archivist.js
index 02384f102..d2fb23225 100644
--- a/plugins/tiddlywiki/twitter-archivist/archivist.js
+++ b/plugins/tiddlywiki/twitter-archivist/archivist.js
@@ -74,14 +74,14 @@ TwitterArchivist.prototype.loadArchive = async function(options) {
 	$tw.utils.each(tweetData,function(tweet) {
 		// Compile the tags for the tweet
 		var tags = ["$:/tags/Tweet"];
-		// Create tiddlers for each mentioned tweeter, and hyperlink the mention in the test
-		var rawText = tweet.tweet.full_text,
-			posText = 0,
-			text = "";
+		// Accumulate the replacements/insertions to the text as an array of {startPos:,endPos:,fnTransform:}
+		var modifications = [];
+		// Mentions
 		var mentions = [];
 		$tw.utils.each(tweet.tweet.entities.user_mentions,function(mention) {
 			var title = "Tweeter - " + mention.id_str;
 			tags.push(title);
+			mentions.push(mention.id_str);
 			wiki.addTiddler({
 				title: title,
 				screenname: "@" + mention.screen_name,
@@ -89,18 +89,53 @@ TwitterArchivist.prototype.loadArchive = async function(options) {
 				user_id: mention.id_str,
 				name: mention.name
 			});
-			text = text + 
-				$tw.utils.htmlEncode(rawText.substring(posText,mention.indices[0])) +
-				"<$link to=\"" + title + "\">" +
-				$tw.utils.htmlEncode(rawText.substring(mention.indices[0],mention.indices[1])) +
-				"</$link>";
-			posText = mention.indices[1];
-			mentions.push(mention.id_str);
+			modifications.push({
+				startPos: mention.indices[0],
+				endPos: mention.indices[1],
+				fnTransform: function(text) {
+					return "<$link to=\"" + title + "\">" +
+						$tw.utils.htmlEncode(text.substring(mention.indices[0],mention.indices[1])) +
+						"</$link>";	
+				}
+			});
 		});
-		if(posText < rawText.length) {
-			text = text + $tw.utils.htmlEncode(rawText.substring(posText));
+		// URLs
+		$tw.utils.each(tweet.tweet.entities.urls,function(urlInfo) {
+			modifications.push({
+				startPos: urlInfo.indices[0],
+				endPos: urlInfo.indices[1],
+				fnTransform: function(text) {
+					return "<a href=\"" + $tw.utils.htmlEncode(urlInfo.expanded_url) + "\" rel=\"noopener noreferrer\" target=\"_blank\">" +
+						$tw.utils.htmlEncode(urlInfo.display_url) +
+						"</a>";	
+				}
+			});
+		});
+		// Sort the modifications by start position
+		modifications.sort(function(a,b) {
+			return a.startPos - b.startPos;
+		});
+		// Apply the modifications in reverse order
+		var rawText = tweet.tweet.full_text,
+			posText = rawText.length - 1,
+			chunks = [];
+		for(var modificationIndex=modifications.length-1; modificationIndex>=0; modificationIndex--) {
+			var modification = modifications[modificationIndex];
+			// Process any text after the modification
+			if(posText > modification.endPos) {
+				chunks.push($tw.utils.htmlEncode(rawText.substring(modification.endPos,posText)));
+			}
+			// Process the modification
+			chunks.push(modification.fnTransform(rawText));
+			// Adjust the position
+			posText = modification.startPos;
 		}
-		text = text.replace("\n","<br>");
+		// Process any remaining text
+		if(posText > 0) {
+			chunks.push($tw.utils.htmlEncode(rawText.substring(0,posText)));
+		}
+		// Concatenate the chunks and replace newlines with <br>
+		var text = chunks.reverse().join("").replace("\n","<br>");
 		// Create the tweet tiddler
 		var tiddler = {
 			title: "Tweet - " + tweet.tweet.id_str,
diff --git a/plugins/tiddlywiki/twitter-archivist/spec.tid b/plugins/tiddlywiki/twitter-archivist/spec.tid
index 7e837c523..8a780a96d 100644
--- a/plugins/tiddlywiki/twitter-archivist/spec.tid
+++ b/plugins/tiddlywiki/twitter-archivist/spec.tid
@@ -28,6 +28,7 @@ The Twitter Archivist imports the following tiddlers:
 |!Field |!Description |
 |''created'' |Tweet creation date (in TiddlyWiki format) |
 |''favorite_count'' |Number of favourites received by this tweet |
+|''mention_user_ids'' |Optional list of user IDs mentioned in the tweet |
 |''modified'' |Tweet creation date (in TiddlyWiki format) |
 |''retweet_count'' |Number of retweets received by this tweet |
 |''status_id'' |Unique numeric identifier for tweet |
diff --git a/plugins/tiddlywiki/twitter-archivist/todo.tid b/plugins/tiddlywiki/twitter-archivist/todo.tid
index af53e677a..62a7912c2 100644
--- a/plugins/tiddlywiki/twitter-archivist/todo.tid
+++ b/plugins/tiddlywiki/twitter-archivist/todo.tid
@@ -4,10 +4,11 @@ title: $:/plugins/tiddlywiki/twitter-archivist/todo
 
 * Fixed display of tweet account information
 * Data model documentation
+* Expand t.co URLs
 
 !! To Do
 
-* Expand t.co URLs
+* Wikify hashtags
 * Import direct messages
 * Control over which media types are imported
 * `_canonical_uri` support for media