Handle directory creation if doesn't exist for test store (#8562)

* fix breaking bug in image tiddler attachment

* fix comments

* fix code format

* refactor processIncomingTiddler flow

* remove whitespaces after if statements

* refactor attachment_blob persistence flow

* refactor process tiddler to support different attachments

* add tests for attachment

* add more attachement test cases

* working on adding instanbul for test coverage report

* code coverage report generation

* remove unnecessary packages

* fix comments

* handle directory creation if doesn't exist for test store

* resolve issue with CI tests failure

editions/test/playwright.spec.js

@@ -12,9 +12,9 @@ test('get started link', async ({ page }) => {
 
     // Load the generated test TW html
     await page.goto(`file:///${crossPlatformIndexPath}`);
-
+    const title = await page.title();    
     // Sanity check
-    await expect(page.locator('.tc-site-title'), "Expected correct page title to verify the test page was loaded").toHaveText('TiddlyWiki5');
+    await expect(title, "Expected correct page title to verify the test page was loaded").toContain('TiddlyWiki5');
 
     // Wait for jasmine results bar to appear
     await expect(page.locator('.jasmine-overall-result'), "Expected jasmine test results bar to be present").toBeVisible({timeout});

package-lock.json

@@ -9,8 +9,10 @@
       "version": "5.3.6-prerelease",
       "license": "BSD",
       "dependencies": {
+        "@playwright/test": "^1.46.1",
         "better-sqlite3": "^9.4.3",
-        "node-sqlite3-wasm": "^0.8.10"
+        "node-sqlite3-wasm": "^0.8.10",
+        "playwright": "^1.46.1"
       },
       "bin": {
         "tiddlywiki": "tiddlywiki.js"
@@ -174,6 +176,20 @@
       "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
       "dev": true
     },
+    "node_modules/@playwright/test": {
+      "version": "1.46.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.46.1.tgz",
+      "integrity": "sha512-Fq6SwLujA/DOIvNC2EL/SojJnkKf/rAwJ//APpJJHRyMi1PdKrY3Az+4XNQ51N4RTbItbIByQ0jgd1tayq1aeA==",
+      "dependencies": {
+        "playwright": "1.46.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/acorn": {
       "version": "7.4.1",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
@@ -793,6 +809,19 @@
       "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
       "dev": true
     },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/functional-red-black-tree": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
@@ -1175,6 +1204,34 @@
         "node": ">=8"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.46.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.46.1.tgz",
+      "integrity": "sha512-oPcr1yqoXLCkgKtD5eNUPLiN40rYEM39odNpIb6VE6S7/15gJmA1NzVv6zJYusV0e7tzvkU/utBFNa/Kpxmwng==",
+      "dependencies": {
+        "playwright-core": "1.46.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.46.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.46.1.tgz",
+      "integrity": "sha512-h9LqIQaAv+CYvWzsZ+h3RsrqCStkBHlgo6/TJlFst3cOTlLghBQlJwPOZKQJTKNaD3QIB7aAVQ+gfWbN3NXB7A==",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/prebuild-install": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",

package.json

@@ -39,7 +39,9 @@
     "lint": "eslint ."
   },
   "dependencies": {
+    "@playwright/test": "^1.46.1",
     "better-sqlite3": "^9.4.3",
-    "node-sqlite3-wasm": "^0.8.10"
+    "node-sqlite3-wasm": "^0.8.10",
+    "playwright": "^1.46.1"
   }
 }

plugins/tiddlywiki/multiwikiserver/modules/tests/test-attachment.js

@@ -6,170 +6,176 @@ tags: [[$:/tags/test-spec]]
 Tests attachments.
 
 \*/
-var fs = require('fs');
+if(typeof window === 'undefined' && typeof process !== 'undefined' && process.versions && process.versions.node) {
-var path = require('path');
+  var fs = require('fs');
-var assert = require('assert');
+  var path = require('path');
-var AttachmentStore = require('$:/plugins/tiddlywiki/multiwikiserver/store/attachments.js').AttachmentStore;
+  var assert = require('assert');
-const {Buffer} = require('buffer');
+  var AttachmentStore = require('$:/plugins/tiddlywiki/multiwikiserver/store/attachments.js').AttachmentStore;
+  const {Buffer} = require('buffer');
 
-function generateFileWithSize(filePath, sizeInBytes) {
+  function generateFileWithSize(filePath, sizeInBytes) {
-  return new Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
-    var buffer = Buffer.alloc(sizeInBytes);
+      var buffer = Buffer.alloc(sizeInBytes);
-    for(var i = 0; i < sizeInBytes; i++) {
+      for(var i = 0; i < sizeInBytes; i++) {
-      buffer[i] = Math.floor(Math.random() * 256);
+        buffer[i] = Math.floor(Math.random() * 256);
-    }
-
-    fs.writeFile(filePath, buffer, (err) => {
-      if(err) {
-        console.error('Error writing file:', err);
-        reject(err);
-      } else {
-        console.log('File '+filePath+' generated with size '+sizeInBytes+' bytes');
-        fs.readFile(filePath, (err, data) => {
-          if(err) {
-            console.error('Error reading file:', err);
-            reject(err);
-          } else {
-            resolve(data);
-          }
-        });
       }
-    });
-  });
-}
 
-(function() {
+      fs.writeFile(filePath, buffer, (err) => {
-'use strict';
+        if(err) {
-if($tw.node) {
+          console.error('Error writing file:', err);
-  describe('AttachmentStore', function() {
+          reject(err);
-    var storePath = './editions/test/test-store';
+        } else {
-    var attachmentStore = new AttachmentStore({ storePath: storePath });
+          console.log('File '+filePath+' generated with size '+sizeInBytes+' bytes');
-    var originalTimeout;
+          fs.readFile(filePath, (err, data) => {
-
+            if(err) {
-    beforeAll(function() {
+              console.error('Error reading file:', err);
-      originalTimeout = jasmine.DEFAULT_TIMEOUT_INTERVAL;
+              reject(err);
-      jasmine.DEFAULT_TIMEOUT_INTERVAL = 50000;
+            } else {
-    });
+              resolve(data);
-
+            }
-    afterAll(function() {
+          });
-      jasmine.DEFAULT_TIMEOUT_INTERVAL = originalTimeout;
-      fs.readdirSync(storePath).forEach(function(file) {
-        var filePath = path.join(storePath, file);
-        if(fs.lstatSync(filePath).isFile()) {
-          fs.unlinkSync(filePath);
-        } else if(fs.lstatSync(filePath).isDirectory()) {
-          fs.rmdirSync(filePath, { recursive: true });
         }
       });
     });
+  }
 
-    it('isValidAttachmentName', function() {
+  (function() {
-      expect(attachmentStore.isValidAttachmentName('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890')).toBe(true);
+  'use strict';
-      expect(attachmentStore.isValidAttachmentName('invalid-name')).toBe(false);
+  if($tw.node) {
-    });
+    describe('AttachmentStore', function() {
+      var storePath = './editions/test/test-store';
+      var attachmentStore = new AttachmentStore({ storePath: storePath });
+      var originalTimeout;
 
-    it('saveAttachment', function() {
+      beforeAll(function() {
-      var options = {
+        const dirPath = path.dirname(`${storePath}/files`);
-        text: 'Hello, World!',
+        if(!fs.existsSync(dirPath)) {
-        type: 'text/plain',
+          fs.mkdirSync(dirPath, { recursive: true });
-        reference: 'test-reference',
+        }
-      };
+        originalTimeout = jasmine.DEFAULT_TIMEOUT_INTERVAL;
-      var contentHash = attachmentStore.saveAttachment(options);
+        jasmine.DEFAULT_TIMEOUT_INTERVAL = 50000;
-      assert.strictEqual(contentHash.length, 64);
+      });
-      assert.strictEqual(fs.existsSync(path.resolve(storePath, 'files', contentHash)), true);
-    });
-  
-    it('adoptAttachment', function() {
-      var incomingFilepath = path.resolve(storePath, 'incoming-file.txt');
-      fs.writeFileSync(incomingFilepath, 'Hello, World!');
-      var type = 'text/plain';
-      var hash = 'abcdef0123456789abcdef0123456789';
-      var _canonical_uri = 'test-canonical-uri';
-      attachmentStore.adoptAttachment(incomingFilepath, type, hash, _canonical_uri);
-      expect(fs.existsSync(path.resolve(storePath, 'files', hash))).toBe(true);
-    });
-  
-    it('getAttachmentStream', function() {
-      var options = {
-        text: 'Hello, World!',
-        type: 'text/plain',
-        filename: 'data.txt',
-      };
-      var contentHash = attachmentStore.saveAttachment(options);
-      var stream = attachmentStore.getAttachmentStream(contentHash);
-      expect(stream).not.toBeNull();
-      expect(stream.type).toBe('text/plain');
-    });
 
-    it('getAttachmentFileSize', function() {
+      afterAll(function() {
-      var options = {
+        jasmine.DEFAULT_TIMEOUT_INTERVAL = originalTimeout;
-        text: 'Hello, World!',
+        fs.readdirSync(storePath).forEach(function(file) {
-        type: 'text/plain',
+          var filePath = path.join(storePath, file);
-        reference: 'test-reference',
+          if(fs.lstatSync(filePath).isFile()) {
-      };
+            fs.unlinkSync(filePath);
-      var contentHash = attachmentStore.saveAttachment(options);
+          } else if(fs.lstatSync(filePath).isDirectory()) {
-      var fileSize = attachmentStore.getAttachmentFileSize(contentHash);
+            fs.rmdirSync(filePath, { recursive: true });
-      expect(fileSize).toBe(13);
+          }
-    });
+        });
+      });
 
-    it('getAttachmentMetadata', function() {
+      it('isValidAttachmentName', function() {
-      var options = {
+        expect(attachmentStore.isValidAttachmentName('abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890')).toBe(true);
-        text: 'Hello, World!',
+        expect(attachmentStore.isValidAttachmentName('invalid-name')).toBe(false);
-        type: 'text/plain',
+      });
-        filename: 'data.txt',
-      };
-      var contentHash = attachmentStore.saveAttachment(options);
-      var metadata = attachmentStore.getAttachmentMetadata(contentHash);
-      expect(metadata).not.toBeNull();
-      expect(metadata.type).toBe('text/plain');
-      expect(metadata.filename).toBe('data.txt');
-    });
 
-    it('saveAttachment large file', async function() {
+      it('saveAttachment', function() {
-      var sizeInMB = 10
-      const file = await generateFileWithSize('./editions/test/test-store/large-file.txt', 1024 * 1024 * sizeInMB)
-      var options = {
-        text: file,
-        type: 'application/octet-stream',
-        reference: 'test-reference',
-      };
-      var contentHash = attachmentStore.saveAttachment(options);
-      assert.strictEqual(contentHash.length, 64);
-      assert.strictEqual(fs.existsSync(path.resolve(storePath, 'files', contentHash)), true);
-    });
-
-    it('saveAttachment multiple large files', async function() {
-      var sizeInMB = 10;
-      var numFiles = 5;
-      for (var i = 0; i < numFiles; i++) {
-        const file = await generateFileWithSize(`./editions/test/test-store/large-file-${i}.txt`, 1024 * 1024 * sizeInMB);
         var options = {
-          text: file,
+          text: 'Hello, World!',
-          type: 'application/octet-stream',
+          type: 'text/plain',
-          reference: `test-reference-${i}`,
+          reference: 'test-reference',
         };
         var contentHash = attachmentStore.saveAttachment(options);
         assert.strictEqual(contentHash.length, 64);
         assert.strictEqual(fs.existsSync(path.resolve(storePath, 'files', contentHash)), true);
-      }
+      });
-    });
+    
-
+      it('adoptAttachment', function() {
-    it('getAttachmentStream multiple large files', async function() {
+        var incomingFilepath = path.resolve(storePath, 'incoming-file.txt');
-      var sizeInMB = 10;
+        fs.writeFileSync(incomingFilepath, 'Hello, World!');
-      var numFiles = 5;
+        var type = 'text/plain';
-      for (var i = 0; i < numFiles; i++) {
+        var hash = 'abcdef0123456789abcdef0123456789';
-        const file = await generateFileWithSize(`./editions/test/test-store/large-file-${i}.txt`, 1024 * 1024 * sizeInMB);
+        var _canonical_uri = 'test-canonical-uri';
+        attachmentStore.adoptAttachment(incomingFilepath, type, hash, _canonical_uri);
+        expect(fs.existsSync(path.resolve(storePath, 'files', hash))).toBe(true);
+      });
+    
+      it('getAttachmentStream', function() {
         var options = {
-          text: file,
+          text: 'Hello, World!',
-          type: 'application/octet-stream',
+          type: 'text/plain',
-          reference: `test-reference-${i}`,
+          filename: 'data.txt',
         };
         var contentHash = attachmentStore.saveAttachment(options);
         var stream = attachmentStore.getAttachmentStream(contentHash);
-        assert.notStrictEqual(stream, null);
+        expect(stream).not.toBeNull();
-        assert.strictEqual(stream.type, 'application/octet-stream');
+        expect(stream.type).toBe('text/plain');
-      }
+      });
+
+      it('getAttachmentFileSize', function() {
+        var options = {
+          text: 'Hello, World!',
+          type: 'text/plain',
+          reference: 'test-reference',
+        };
+        var contentHash = attachmentStore.saveAttachment(options);
+        var fileSize = attachmentStore.getAttachmentFileSize(contentHash);
+        expect(fileSize).toBe(13);
+      });
+
+      it('getAttachmentMetadata', function() {
+        var options = {
+          text: 'Hello, World!',
+          type: 'text/plain',
+          filename: 'data.txt',
+        };
+        var contentHash = attachmentStore.saveAttachment(options);
+        var metadata = attachmentStore.getAttachmentMetadata(contentHash);
+        expect(metadata).not.toBeNull();
+        expect(metadata.type).toBe('text/plain');
+        expect(metadata.filename).toBe('data.txt');
+      });
+
+      it('saveAttachment large file', async function() {
+        var sizeInMB = 10
+        const file = await generateFileWithSize('./editions/test/test-store/large-file.txt', 1024 * 1024 * sizeInMB)
+        var options = {
+          text: file,
+          type: 'application/octet-stream',
+          reference: 'test-reference',
+        };
+        var contentHash = attachmentStore.saveAttachment(options);
+        assert.strictEqual(contentHash.length, 64);
+        assert.strictEqual(fs.existsSync(path.resolve(storePath, 'files', contentHash)), true);
+      });
+
+      it('saveAttachment multiple large files', async function() {
+        var sizeInMB = 10;
+        var numFiles = 5;
+        for (var i = 0; i < numFiles; i++) {
+          const file = await generateFileWithSize(`./editions/test/test-store/large-file-${i}.txt`, 1024 * 1024 * sizeInMB);
+          var options = {
+            text: file,
+            type: 'application/octet-stream',
+            reference: `test-reference-${i}`,
+          };
+          var contentHash = attachmentStore.saveAttachment(options);
+          assert.strictEqual(contentHash.length, 64);
+          assert.strictEqual(fs.existsSync(path.resolve(storePath, 'files', contentHash)), true);
+        }
+      });
+
+      it('getAttachmentStream multiple large files', async function() {
+        var sizeInMB = 10;
+        var numFiles = 5;
+        for (var i = 0; i < numFiles; i++) {
+          const file = await generateFileWithSize(`./editions/test/test-store/large-file-${i}.txt`, 1024 * 1024 * sizeInMB);
+          var options = {
+            text: file,
+            type: 'application/octet-stream',
+            reference: `test-reference-${i}`,
+          };
+          var contentHash = attachmentStore.saveAttachment(options);
+          var stream = attachmentStore.getAttachmentStream(contentHash);
+          assert.notStrictEqual(stream, null);
+          assert.strictEqual(stream.type, 'application/octet-stream');
+        }
+      });
     });
-  });
+  }
+  })();
 }
-})();