From 2d06d918a154c15196ca92fb8f7873ca3c797f00 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 02:21:31 +0900 Subject: Timeout, package and docs cleanup --- README.md | 17 ++++++----------- docs/migrating.md | 14 ++++++++++++++ docs/pm2.md | 11 ----------- package.json | 5 ++--- src/api/structures/Server.js | 3 ++- src/setup.js | 2 -- 6 files changed, 24 insertions(+), 28 deletions(-) create mode 100644 docs/migrating.md delete mode 100644 docs/pm2.md diff --git a/README.md b/README.md index 6c9433f..0f531c3 100644 --- a/README.md +++ b/README.md @@ -8,33 +8,28 @@ This guide asumes a lot of things, including that you know your way around linux, nginx and internet in general. - Decently updated version of linux -- `node` package installed and at least at version 10 -- `build-essential` package installed to build some dependencies -- `ffmpeg` package installed if you want thumbnails +- `node` version 12+ +- `build-essential` package installed to build dependencies +- `ffmpeg` package installed if you want video thumbnails - `yarn` package installed. If you'd like to use npm instead change `package.json` accordingly -- A database, postgresql preferably. You can also fall back to sqlite3 by default. +- `pm2` globally installed (`npm i -g pm2`) to keep the service alive at all times. +- A database, postgresql preferably. You can also fall back to sqlite3 which ships by default. ### Installing 1. Clone the repository and `cd` into it 2. Run `yarn install` 3. Run `yarn setup` -4. Run `yarn migrate` -5. Run `yarn seed` Lolisafe is now installed, configured and ready. Now you need to serve it to the public by using a domain name. 6. Check the [nginx](docs/nginx.md) file for a sample configuration that has every step to run lolisafe securely on production. -After you finish setting up nginx, you need to start lolisafe by using pm2. If you want to use something else, figure out how. (More info on why pm2 [here](docs/pm2.md)) +After you finish setting up nginx, you need to start lolisafe by using pm2. If you want to use something else like forever, ensure that the process spawned from `npm run start` never dies. 7. Run `pm2 start pm2.json`: 8. Profit -### Cloudflare - -If you want to run your site through CloudFlare because of the obvious advantages it has, lolisafe has your back. Unless you manually modify the `.env` file, uploads through the website will be uploaded in chunks thus bypassing CloudFlare's 100mb upload limit per file. - ## Author **lolisafe** © [Pitu](https://github.com/Pitu), Released under the [MIT](https://github.com/WeebDev/lolisafe/blob/master/LICENSE) License.
diff --git a/docs/migrating.md b/docs/migrating.md new file mode 100644 index 0000000..a7d9eb7 --- /dev/null +++ b/docs/migrating.md @@ -0,0 +1,14 @@ +### Migrate from v3 to v4 +This version introduces a few breaking changes and updating requires some manual work. +For starters we recommend cloning the new version somewhere else instead of `git pull` on your v3 version. + +- After cloning move your `uploads` folder from the v3 folder to the new v4 folder. +- Then copy your `database/db` file from your v3 folder to the root of your v4 folder. +- You then need to run `yarn setup` or `npm start setup` from the v4 folder and finish the setup process. +- Once that's done you need to manually run `node src/api/databaseMigration.js` from the root folder of v4 +- After the migration finishes, the last step is to update your nginx config with the [newly provided script](./nginx.md) +- Restart nginx with `sudo nginx -s reload` +- And lastly start your lolisafe instance with `pm2 start pm2.json` + +### Known issues of migrating +- The thumbnails in the album view don't show up. That's because they don't exist, this will get solved as you upload new stuff so the newly uploaded files get the proper thumbnail created. diff --git a/docs/pm2.md b/docs/pm2.md deleted file mode 100644 index ac942fc..0000000 --- a/docs/pm2.md +++ /dev/null @@ -1,11 +0,0 @@ -## Setting up PM2 to run lolisafe - -The best way to keep the service running in case of crashes or unexpected issues is to attach the process to PM2 and forget about it. This also gives you the ability to dettach the process from your terminal and run it in the background, which is a must since lolisafe now comes in 2 separate processes. -The recommended way to set it up is to run the commands below, one for the API and the other for the site. - -``` -pm2 start npm --name "lolisafe-api" -- run api -pm2 start npm --name "lolisafe-site" -- run site -``` - -All set, if you want to check the logs you can `pm2 logs lolisafe-api` or similar. diff --git a/package.json b/package.json index 41ad042..200047d 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "email": "heyitspitu@gmail.com", "url": "https://github.com/Pitu" }, - "main": "src/_scripts/start.js", "scripts": { "setup": "node src/setup.js && yarn build && yarn migrate && yarn seed", "build": "nuxt build", @@ -18,7 +17,7 @@ "seed": "yarn knex seed:run", "api": "node src/api/structures/Server", "update": "git pull && yarn install && yarn migrate && yarn build && yarn restart", - "restart": "pm2 restart lolisafe-api && pm2 restart lolisafe-website" + "restart": "pm2 restart lolisafe" }, "repository": { "type": "git", @@ -28,7 +27,7 @@ "url": "https://github.com/WeebDev/lolisafe/issues" }, "engines": { - "node": ">=8.0.0" + "node": ">=12.0.0" }, "dependencies": { "@nuxtjs/axios": "^5.4.1", diff --git a/src/api/structures/Server.js b/src/api/structures/Server.js index a8eccd9..2039ed5 100644 --- a/src/api/structures/Server.js +++ b/src/api/structures/Server.js @@ -78,9 +78,10 @@ class Server { jetpack.dir('uploads/thumbs/square'); this.registerAllTheRoutes(); this.serveNuxt(); - this.server.listen(this.port, () => { + const server = this.server.listen(this.port, () => { log.success(`Backend ready and listening on port ${this.port}`); }); + server.setTimeout(600000); } } diff --git a/src/setup.js b/src/setup.js index 080320f..a02718a 100644 --- a/src/setup.js +++ b/src/setup.js @@ -155,8 +155,6 @@ async function start() { console.log('============================================='); console.log('== .env file generated successfully. =='); console.log('============================================='); - console.log('== Run `yarn migrate` and `yarn seed` next =='); - console.log('============================================='); console.log(); } -- cgit v1.2.3 From 4dafc79cb74d901bb9454f78277298f020543bb5 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 02:55:05 +0900 Subject: fix authorization --- docs/migrating.md | 4 ++++ src/api/utils/Util.js | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/migrating.md b/docs/migrating.md index a7d9eb7..796d422 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -12,3 +12,7 @@ For starters we recommend cloning the new version somewhere else instead of `git ### Known issues of migrating - The thumbnails in the album view don't show up. That's because they don't exist, this will get solved as you upload new stuff so the newly uploaded files get the proper thumbnail created. + +### Breaking changes +- You need to update the lolisafe browser if you use it, since it won't work with the new version automatically. Instead of pasting your token into it, you need to log in to lolisafe, go to your user settings and generate an `API KEY`, which you will use to access the service from 3rd party apps like the browser extension, ShareX, etc. +- To upload a file to an album directly users used to use the endpoint `/api/upload/${albumId}`. This is no longer the case. To upload directly to an album now it's necessary to pass a header called `albumid` with an integer as the value of the album to which you want to upload a file to. diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index b8d960d..80bffd5 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -206,7 +206,15 @@ class Util { } } - static isAuthorized(req) { + static async isAuthorized(req) { + if (req.headers.token) { + if (!this.options.canApiKey) return false; + const user = await db.table('users').where({ apiKey: req.headers.token }).first(); + if (!user) return false; + if (!user.enabled) return false; + return true; + } + if (!req.headers.authorization) return false; const token = req.headers.authorization.split(' ')[1]; if (!token) return false; -- cgit v1.2.3 From 5f58431409e1a4e875cd8121cfe9dc47cfecc65e Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 02:57:24 +0900 Subject: Fix authorization --- src/api/utils/Util.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 80bffd5..7f6dd22 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -208,10 +208,8 @@ class Util { static async isAuthorized(req) { if (req.headers.token) { - if (!this.options.canApiKey) return false; const user = await db.table('users').where({ apiKey: req.headers.token }).first(); - if (!user) return false; - if (!user.enabled) return false; + if (!user || !user.enabled) return false; return true; } -- cgit v1.2.3 From 407fb8bcc31cd69394a2444db53b710cc2dc4d55 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:05:12 +0900 Subject: Fix authorization --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 7f6dd22..91ab663 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -210,7 +210,7 @@ class Util { if (req.headers.token) { const user = await db.table('users').where({ apiKey: req.headers.token }).first(); if (!user || !user.enabled) return false; - return true; + return user; } if (!req.headers.authorization) return false; -- cgit v1.2.3 From a057f26896d98e8a99819b5de62c79cccb2ec023 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:23:59 +0900 Subject: Fix url retrieval of uploaded file --- src/api/routes/uploads/uploadPOST.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/routes/uploads/uploadPOST.js b/src/api/routes/uploads/uploadPOST.js index 6c01dd3..4b84da6 100644 --- a/src/api/routes/uploads/uploadPOST.js +++ b/src/api/routes/uploads/uploadPOST.js @@ -122,6 +122,7 @@ class uploadPOST extends Route { this.saveFileToAlbum(db, albumId, insertedId); } + uploadedFile = Util.constructFilePublicLink(uploadedFile); return res.status(201).send({ message: 'Sucessfully uploaded the file.', ...uploadedFile @@ -130,6 +131,7 @@ class uploadPOST extends Route { } fileExists(res, exists, filename) { + exists = Util.constructFilePublicLink(exists); res.json({ message: 'Successfully uploaded the file.', name: exists.name, -- cgit v1.2.3 From 6dd7500084bf7306f66e8f65367b90f1049e3f15 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:35:08 +0900 Subject: Fix deleting files without thumb --- src/api/utils/Util.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 91ab663..4674dde 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -154,8 +154,6 @@ class Util { const thumbName = this.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } @@ -163,6 +161,14 @@ class Util { log.error(`There was an error removing the file < ${filename} >`); log.error(error); } + + try { + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); + } catch (error) { + log.error(`There was an error removing the thumbs for file < ${filename} >`); + log.error(error); + } } static async deleteAllFilesFromAlbum(id) { -- cgit v1.2.3 From b70a75da1af00e932319b3ba24ccde860a6c7f48 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:37:52 +0900 Subject: Fix for real --- src/api/utils/Util.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 4674dde..885228f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -154,6 +154,13 @@ class Util { const thumbName = this.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + if (thumbName) { + const thumb = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName); + const thumbSquare = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName); + if (await jetpack.existsAsync(thumb)) jetpack.removeAsync(thumb); + if (await jetpack.existsAsync(thumbSquare)) jetpack.removeAsync(thumbSquare); + } + if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } @@ -161,14 +168,6 @@ class Util { log.error(`There was an error removing the file < ${filename} >`); log.error(error); } - - try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); - } catch (error) { - log.error(`There was an error removing the thumbs for file < ${filename} >`); - log.error(error); - } } static async deleteAllFilesFromAlbum(id) { -- cgit v1.2.3 From ec51cc803e3a30bd2d414aba1a36a979c2a53744 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:40:25 +0900 Subject: Update TODO --- TODO | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index 1c84982..70b72e9 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,4 @@ - There's a vertical scrollbar on seemingly all pages even though there is no need for it -- Passwords that contain special characters don't seem to work (refer to previous dms) - Uploaded text file (via ShareX) does not show up in the dashboard's "Files" area * Currently only pictures show up on the dashboard due to having thumbs @@ -9,3 +8,5 @@ - Think of a strategy to achieve this in a nice manner - Can't delete/rename albums when going into album view. Or ever. +- Dark theme the annoying popups for deleting and other stuff +- Logout button -- cgit v1.2.3 From 8ffa0ba075a9d2b7b7409f9d11581a5237e7fd89 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 04:31:12 +0900 Subject: Add endpoint with version of the API --- src/api/routes/service/versionGET.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/api/routes/service/versionGET.js diff --git a/src/api/routes/service/versionGET.js b/src/api/routes/service/versionGET.js new file mode 100644 index 0000000..dfb994a --- /dev/null +++ b/src/api/routes/service/versionGET.js @@ -0,0 +1,15 @@ +const Route = require('../../structures/Route'); + +class versionGET extends Route { + constructor() { + super('/version', 'get', { bypassAuth: true }); + } + + run(req, res) { + return res.json({ + version: process.env.npm_package_version + }); + } +} + +module.exports = versionGET; -- cgit v1.2.3 From 5e07436f0d4ce0b22f37bf7c8c05ac467537a7fa Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 04:45:58 +0900 Subject: Add logout button --- src/site/components/navbar/Navbar.vue | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/site/components/navbar/Navbar.vue b/src/site/components/navbar/Navbar.vue index 0b98af0..48dfeb1 100644 --- a/src/site/components/navbar/Navbar.vue +++ b/src/site/components/navbar/Navbar.vue @@ -45,6 +45,12 @@ exact> Account + + Logout +
+
this.filesOffsetEndWaterfall; + return this.files.length > this.filesOffsetEnd; } }, methods: { - loadMoreFilesWaterfall() { - this.filesOffsetEndWaterfall = this.filesOffsetEndWaterfall + this.filesPerPageWaterfall; + loadMoreFiles() { + this.filesOffsetEnd = this.filesOffsetEnd + this.filesPerPage; }, async search() { const data = await this.$search.do(this.searchTerm, [ -- cgit v1.2.3 From d644b21d431c03263aa7191fe54a984aac96f979 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 05:50:47 +0900 Subject: Make thumbnails webp (bye bye safari) --- src/api/utils/Util.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 885228f..a4af81e 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -36,7 +36,7 @@ class Util { static generateThumbnails(filename) { const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; + const output = `${filename.slice(0, -ext.length)}.webp`; if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); return null; @@ -46,11 +46,11 @@ class Util { const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); await sharp(file) .resize(64, 64) - .toFormat('png') + .toFormat('webp') .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); await sharp(file) .resize(225, null) - .toFormat('png') + .toFormat('webp') .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); } @@ -76,8 +76,9 @@ class Util { static getFileThumbnail(filename) { if (!filename) return null; const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; + const extension = imageExtensions.includes(ext) ? 'webp' : videoExtensions.includes(ext) ? 'png' : null; + if (!extension) return null; + return `${filename.slice(0, -ext.length)}.${extension}`; } static constructFilePublicLink(file) { -- cgit v1.2.3 From 898a2dde78ae2ed352eec2b845fc09ecda864451 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 05:50:59 +0900 Subject: Re-process thumbnails on migration --- src/api/databaseMigration.js | 52 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/api/databaseMigration.js b/src/api/databaseMigration.js index 5cf4b39..06d3849 100644 --- a/src/api/databaseMigration.js +++ b/src/api/databaseMigration.js @@ -1,5 +1,9 @@ const nodePath = require('path'); const moment = require('moment'); +const jetpack = require('fs-jetpack'); +const { path } = require('fs-jetpack'); +const sharp = require('sharp'); +const ffmpeg = require('fluent-ffmpeg'); const oldDb = require('knex')({ client: 'sqlite3', @@ -42,6 +46,10 @@ const start = async () => { console.log('Starting migration, this may take a few minutes...'); // Because I half assed it console.log('Please do NOT kill the process. Wait for it to finish.'); + await jetpack.removeAsync(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs')); + await jetpack.dirAsync(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square')); + console.log('Finished deleting old thumbnails to create new ones'); + const users = await oldDb.table('users').where('username', '<>', 'root'); for (const user of users) { const now = moment.utc().toDate(); @@ -113,6 +121,7 @@ const start = async () => { albumId: file.albumid, fileId: file.id }); + generateThumbnails(file.name); } await newDb.batchInsert('files', filesToInsert, 20); await newDb.batchInsert('albumsFiles', albumsFilesToInsert, 20); @@ -122,4 +131,47 @@ const start = async () => { process.exit(0); }; +const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; +const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; + +const generateThumbnails = filename => { + if (!jetpack.exists(nodePath.join(__dirname, '..', '..', 'uploads', filename))) return; + + const ext = nodePath.extname(filename).toLowerCase(); + const output = `${filename.slice(0, -ext.length)}.webp`; + if (imageExtensions.includes(ext)) return generateThumbnailForImage(filename, output); + if (videoExtensions.includes(ext)) return generateThumbnailForVideo(filename); +}; + +const generateThumbnailForImage = async (filename, output) => { + const file = await jetpack.readAsync(nodePath.join(__dirname, '..', '..', 'uploads', filename), 'buffer'); + await sharp(file) + .resize(64, 64) + .toFormat('webp') + .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square', output)); + await sharp(file) + .resize(225, null) + .toFormat('webp') + .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', output)); +}; + +const generateThumbnailForVideo = filename => { + ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square'), + size: '64x64' + }) + .on('error', error => console.error(error.message)); + ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs'), + size: '150x?' + }) + .on('error', error => console.error(error.message)); +}; + start(); -- cgit v1.2.3 From 8e3c3841a4d0f1a6bc71039f94e030b8526105e8 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:39:24 +0900 Subject: Update database migration --- src/api/databaseMigration.js | 83 ++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/src/api/databaseMigration.js b/src/api/databaseMigration.js index 06d3849..15b2bf7 100644 --- a/src/api/databaseMigration.js +++ b/src/api/databaseMigration.js @@ -5,6 +5,9 @@ const { path } = require('fs-jetpack'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); +const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; +const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; + const oldDb = require('knex')({ client: 'sqlite3', connection: { @@ -121,7 +124,13 @@ const start = async () => { albumId: file.albumid, fileId: file.id }); - generateThumbnails(file.name); + + const filename = file.name; + if (!jetpack.exists(nodePath.join(__dirname, '..', '..', 'uploads', filename))) continue; + const ext = nodePath.extname(filename).toLowerCase(); + const output = `${filename.slice(0, -ext.length)}.webp`; + if (imageExtensions.includes(ext)) await generateThumbnailForImage(filename, output); + if (videoExtensions.includes(ext)) generateThumbnailForVideo(filename); } await newDb.batchInsert('files', filesToInsert, 20); await newDb.batchInsert('albumsFiles', albumsFilesToInsert, 20); @@ -131,47 +140,45 @@ const start = async () => { process.exit(0); }; -const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; - -const generateThumbnails = filename => { - if (!jetpack.exists(nodePath.join(__dirname, '..', '..', 'uploads', filename))) return; - - const ext = nodePath.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.webp`; - if (imageExtensions.includes(ext)) return generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return generateThumbnailForVideo(filename); -}; - const generateThumbnailForImage = async (filename, output) => { - const file = await jetpack.readAsync(nodePath.join(__dirname, '..', '..', 'uploads', filename), 'buffer'); - await sharp(file) - .resize(64, 64) - .toFormat('webp') - .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square', output)); - await sharp(file) - .resize(225, null) - .toFormat('webp') - .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', output)); + try { + const file = await jetpack.readAsync(nodePath.join(__dirname, '..', '..', 'uploads', filename), 'buffer'); + await sharp(file) + .resize(64, 64) + .toFormat('webp') + .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square', output)); + await sharp(file) + .resize(225, null) + .toFormat('webp') + .toFile(nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', output)); + console.log('finished', filename); + } catch (error) { + console.log('error', filename); + } }; const generateThumbnailForVideo = filename => { - ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square'), - size: '64x64' - }) - .on('error', error => console.error(error.message)); - ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs'), - size: '150x?' - }) - .on('error', error => console.error(error.message)); + try { + ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs', 'square'), + size: '64x64' + }) + .on('error', error => console.error(error.message)); + ffmpeg(nodePath.join(__dirname, '..', '..', 'uploads', filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: nodePath.join(__dirname, '..', '..', 'uploads', 'thumbs'), + size: '150x?' + }) + .on('error', error => console.error(error.message)); + console.log('finished', filename); + } catch (error) { + console.log('error', filename); + } }; start(); -- cgit v1.2.3 From 91507249c040e34a2eab878b6e40dcabe7710a5c Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:40:21 +0900 Subject: Update readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0f531c3..452ae6f 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ [![Support me](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fpitu&style=flat-square)](https://www.patreon.com/pitu) [![Support me](https://img.shields.io/badge/Support-Buy%20me%20a%20coffee-yellow.svg?style=flat-square)](https://www.buymeacoffee.com/kana) +### Attention +If you are upgrading from v3 to v4 (current release) and you want to keep your files and relations please read the [migration guide](docs/migration.md). + ### Pre-requisites This guide asumes a lot of things, including that you know your way around linux, nginx and internet in general. -- cgit v1.2.3 From d3b7321bf6ce651721d7081ec2740bd5df3a6746 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:41:08 +0900 Subject: whoops --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 452ae6f..330a542 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Support me](https://img.shields.io/badge/Support-Buy%20me%20a%20coffee-yellow.svg?style=flat-square)](https://www.buymeacoffee.com/kana) ### Attention -If you are upgrading from v3 to v4 (current release) and you want to keep your files and relations please read the [migration guide](docs/migration.md). +If you are upgrading from v3 to v4 (current release) and you want to keep your files and relations please read the [migration guide](docs/migrating.md). ### Pre-requisites This guide asumes a lot of things, including that you know your way around linux, nginx and internet in general. -- cgit v1.2.3 From aea442a9569a91b9610545978131f8dec0ab4ff3 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:45:01 +0900 Subject: Update migration instructions --- docs/migrating.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/migrating.md b/docs/migrating.md index 796d422..6421391 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -5,14 +5,16 @@ For starters we recommend cloning the new version somewhere else instead of `git - After cloning move your `uploads` folder from the v3 folder to the new v4 folder. - Then copy your `database/db` file from your v3 folder to the root of your v4 folder. - You then need to run `yarn setup` or `npm start setup` from the v4 folder and finish the setup process. -- Once that's done you need to manually run `node src/api/databaseMigration.js` from the root folder of v4 -- After the migration finishes, the last step is to update your nginx config with the [newly provided script](./nginx.md) -- Restart nginx with `sudo nginx -s reload` -- And lastly start your lolisafe instance with `pm2 start pm2.json` +- Once that's done you need to manually run `node src/api/databaseMigration.js` from the root folder of v4. +- This will migrate the v3 database to v4 and regenerate every single thumbnail to acomodate the new format. +- After the migration finishes, the last step is to update your nginx config with the [newly provided script](./nginx.md). +- Restart nginx with `sudo nginx -s reload`. +- And lastly start your lolisafe instance with `pm2 start pm2.json`. ### Known issues of migrating - The thumbnails in the album view don't show up. That's because they don't exist, this will get solved as you upload new stuff so the newly uploaded files get the proper thumbnail created. ### Breaking changes -- You need to update the lolisafe browser if you use it, since it won't work with the new version automatically. Instead of pasting your token into it, you need to log in to lolisafe, go to your user settings and generate an `API KEY`, which you will use to access the service from 3rd party apps like the browser extension, ShareX, etc. -- To upload a file to an album directly users used to use the endpoint `/api/upload/${albumId}`. This is no longer the case. To upload directly to an album now it's necessary to pass a header called `albumid` with an integer as the value of the album to which you want to upload a file to. +- If you are using the lolisafe extension from one of the stores, the new version has been submitted and could take up to a week to get approved. In the meantime you can load the unpacked extension by cloning [this repo](https://github.com/WeebDev/loli-safe-extension). +- The lolisafe browser extension needs your new token. Instead of pasting your jwt token into it like before, you need to log in to lolisafe, go to your user settings and generate an `API KEY`, which you will use to access the service from 3rd party apps like the browser extension, ShareX, etc. +- To upload a file to an album directly users used to use the endpoint `/api/upload/${albumId}`. This is no longer the case. To upload directly to an album now it's necessary to pass a header called `albumid` with an integer as the value of the album to which you want to upload the file to. -- cgit v1.2.3 From 0b6867f1b1f490320126368304f6aaaca6807422 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:45:51 +0900 Subject: update guide --- docs/migrating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migrating.md b/docs/migrating.md index 6421391..ffd61ab 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -6,7 +6,7 @@ For starters we recommend cloning the new version somewhere else instead of `git - Then copy your `database/db` file from your v3 folder to the root of your v4 folder. - You then need to run `yarn setup` or `npm start setup` from the v4 folder and finish the setup process. - Once that's done you need to manually run `node src/api/databaseMigration.js` from the root folder of v4. -- This will migrate the v3 database to v4 and regenerate every single thumbnail to acomodate the new format. +- This will migrate the v3 database to v4 and regenerate every single thumbnail in webp to save bandwidth. - After the migration finishes, the last step is to update your nginx config with the [newly provided script](./nginx.md). - Restart nginx with `sudo nginx -s reload`. - And lastly start your lolisafe instance with `pm2 start pm2.json`. -- cgit v1.2.3 From 37a5a8cce220a84ca151a1041b9aeaab2855f771 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:50:39 +0900 Subject: Upgrade migration doc --- docs/migrating.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/migrating.md b/docs/migrating.md index ffd61ab..9166581 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -11,9 +11,6 @@ For starters we recommend cloning the new version somewhere else instead of `git - Restart nginx with `sudo nginx -s reload`. - And lastly start your lolisafe instance with `pm2 start pm2.json`. -### Known issues of migrating -- The thumbnails in the album view don't show up. That's because they don't exist, this will get solved as you upload new stuff so the newly uploaded files get the proper thumbnail created. - ### Breaking changes - If you are using the lolisafe extension from one of the stores, the new version has been submitted and could take up to a week to get approved. In the meantime you can load the unpacked extension by cloning [this repo](https://github.com/WeebDev/loli-safe-extension). - The lolisafe browser extension needs your new token. Instead of pasting your jwt token into it like before, you need to log in to lolisafe, go to your user settings and generate an `API KEY`, which you will use to access the service from 3rd party apps like the browser extension, ShareX, etc. -- cgit v1.2.3 From bf63bc5e2eb844a771432185e25f8dd6b8360e9f Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 16:58:29 +0900 Subject: Update setup script --- src/setup.js | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/setup.js b/src/setup.js index a02718a..212dd0d 100644 --- a/src/setup.js +++ b/src/setup.js @@ -14,14 +14,9 @@ async function start() { const wizard = [ { type: 'input', - query: 'Port to run the API in:', + query: 'Port to run lolisafe in:', handle: 'SERVER_PORT' }, - { - type: 'input', - query: 'Port to run the Website in when in dev mode:', - handle: 'WEBSITE_PORT' - }, { type: 'input', query: 'Full domain this instance is gonna be running on (Ex: https://lolisafe.moe):', @@ -37,13 +32,6 @@ async function start() { query: 'Maximum allowed upload file size in MB (Ex: 100):', handle: 'MAX_SIZE' }, - { - type: 'confirm', - query: 'Generate thumbnails for images/videos? (Requires ffmpeg installed and in your PATH)', - handle: 'GENERATE_THUMBNAILS', - accept: 'y', - deny: 'n' - }, { type: 'confirm', query: 'Allow users to download entire albums in ZIP format?', @@ -53,31 +41,14 @@ async function start() { }, { type: 'confirm', - query: 'Serve files with node?', - handle: 'SERVE_WITH_NODE', - accept: 'y', - deny: 'n' - }, - { - type: 'input', - query: 'Base number of characters for generated file URLs (12 should be good enough):', - handle: 'GENERATED_FILENAME_LENGTH' - }, - { - type: 'input', - query: 'Base number of characters for generated album URLs (6 should be enough):', - handle: 'GENERATED_ALBUM_LENGTH' - }, - { - type: 'confirm', - query: 'Run lolisafe in public mode? (People will be able to upload without an account)', + query: 'Allow people to upload files without an account?', handle: 'PUBLIC_MODE', accept: 'y', deny: 'n' }, { type: 'confirm', - query: 'Enable user signup for new accounts?', + query: 'Allow people to create new accounts?', handle: 'USER_ACCOUNTS', accept: 'y', deny: 'n' @@ -129,6 +100,11 @@ async function start() { let envfile = ''; const defaultSettings = { + GENERATED_FILENAME_LENGTH: 12, + GENERATED_ALBUM_LENGTH: 6, + WEBSITE_PORT: 5001, + SERVE_WITH_NODE: true, + GENERATE_THUMBNAILS: true, CHUNK_SIZE: 90, ROUTE_PREFIX: '/api', RATE_LIMIT_WINDOW: 2, -- cgit v1.2.3