From 42f1a1003a299ca4e571d503d7200b76ff41e752 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:11:51 +0300 Subject: feat: externalize thumb generation function for easier testing --- src/api/utils/Util.js | 118 ++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 67 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 07c295d..8c9428d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -14,15 +14,13 @@ const db = require('knex')({ useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false }); const moment = require('moment'); -const log = require('../utils/Log'); const crypto = require('crypto'); -const sharp = require('sharp'); -const ffmpeg = require('fluent-ffmpeg'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); -const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; +const log = require('./Log'); +const ThumbUtil = require('./ThumbUtil'); + const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { @@ -34,59 +32,13 @@ class Util { return blockedExtensions.includes(extension); } - static generateThumbnails(filename) { - const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; - if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); - return null; - } - - static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); - await sharp(file) - .resize(64, 64) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); - await sharp(file) - .resize(225, null) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); - } - - static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), - size: '64x64' - }) - .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), - size: '150x?' - }) - .on('error', error => log.error(error.message)); - } - - static getFileThumbnail(filename) { - if (!filename) return null; - const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; - } - static constructFilePublicLink(file) { /* TODO: This wont work without a reverse proxy serving both the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = this.getFileThumbnail(file.name); + const thumb = ThumbUtil.getFileThumbnail(file.name); if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; @@ -96,10 +48,11 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = + randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase' + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); @@ -117,7 +70,10 @@ class Util { length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), capitalization: 'lowercase' }); - const exists = await db.table('links').where({ identifier }).first(); + const exists = await db + .table('links') + .where({ identifier }) + .first(); if (!exists) return identifier; /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop @@ -130,7 +86,10 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); + const file = await jetpack.readAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), + 'buffer' + ); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -142,7 +101,10 @@ class Util { } static generateFileHash(data) { - const hash = crypto.createHash('md5').update(data).digest('hex'); + const hash = crypto + .createHash('md5') + .update(data) + .digest('hex'); return hash; } @@ -151,13 +113,20 @@ class Util { } static async deleteFile(filename, deleteFromDB = false) { - const thumbName = this.getFileThumbnail(filename); + const thumbName = ThumbUtil.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) + ); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) + ); if (deleteFromDB) { - await db.table('files').where('name', filename).delete(); + await db + .table('files') + .where('name', filename) + .delete(); } } catch (error) { log.error(`There was an error removing the file < ${filename} >`); @@ -169,7 +138,8 @@ class Util { try { const fileAlbums = await db.table('albumsFiles').where({ albumId: id }); for (const fileAlbum of fileAlbums) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileAlbum.fileId }) .first(); if (!file) continue; @@ -195,7 +165,8 @@ class Util { try { const fileTags = await db.table('fileTags').where({ tagId: id }); for (const fileTag of fileTags) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileTag.fileId }) .first(); if (!file) continue; @@ -219,7 +190,10 @@ class Util { const id = decoded ? decoded.sub : ''; const iat = decoded ? decoded.iat : ''; - const user = await db.table('users').where({ id }).first(); + const user = await db + .table('users') + .where({ id }) + .first(); if (!user || !user.enabled) return false; if (iat && iat < moment(user.passwordEditedAt).format('x')) return false; @@ -233,7 +207,17 @@ class Util { for (const file of files) { zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); } - zip.writeZip(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'zips', `${album.userId}-${album.id}.zip`)); + zip.writeZip( + path.join( + __dirname, + '..', + '..', + '..', + process.env.UPLOAD_FOLDER, + 'zips', + `${album.userId}-${album.id}.zip` + ) + ); } catch (error) { log.error(error); } -- cgit v1.2.3 From a790d7749e04d71df1613e6a02258982683aa290 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:42:20 +0300 Subject: feat: add experimental meaningful preview extraction from videos For now, it sitll requires gifski. It could be rewritten to use webp instead of gifs, because that is a lot faster, uses less space and we could use ffmpeg for it. --- src/api/utils/Util.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 8c9428d..c997581 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -24,6 +24,8 @@ const ThumbUtil = require('./ThumbUtil'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { + static uploadPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER); + static uuid() { return uuidv4(); } @@ -55,7 +57,7 @@ class Util { }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem - const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + const exists = jetpack.exists(path.join(Util.uploadPath, filename)); if (!exists) return filename; if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for file'); @@ -86,10 +88,7 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), - 'buffer' - ); + const file = await jetpack.readAsync(path.join(Util.uploadPath, filename), 'buffer'); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -115,13 +114,9 @@ class Util { static async deleteFile(filename, deleteFromDB = false) { const thumbName = ThumbUtil.getFileThumbnail(filename); try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) - ); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) - ); + await jetpack.removeAsync(path.join(Util.uploadPath, filename)); + await ThumbUtil.removeThumbs(thumbName); + if (deleteFromDB) { await db .table('files') @@ -205,7 +200,7 @@ class Util { try { const zip = new Zip(); for (const file of files) { - zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + zip.addLocalFile(path.join(Util.uploadPath, file)); } zip.writeZip( path.join( -- cgit v1.2.3 From 1e1f3fbb27976a34f53a4e8d250da34dad4e6c20 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sat, 4 Jul 2020 23:18:51 +0300 Subject: feat: experimental videos in grid --- src/api/utils/Util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index c997581..496ea18 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -40,10 +40,11 @@ class Util { the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = ThumbUtil.getFileThumbnail(file.name); + const { thumb, preview } = ThumbUtil.getFileThumbnail(file.name) || {}; if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; + file.preview = preview && `${process.env.DOMAIN}/thumbs/preview/${preview}`; } return file; } -- cgit v1.2.3 From eccbb1ca93f1b86e9bc93dcbc1ec0ee9b168d949 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 02:32:12 +0300 Subject: fix: errors in Util caused by separating into different classes improperly --- src/api/utils/Util.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 496ea18..ab59c95 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); @@ -9,9 +10,9 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: path.join(__dirname, '..', '..', '..', 'database.sqlite') + filename: path.join(__dirname, '../../../database.sqlite'), }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false + useNullAsDefault: process.env.DB_CLIENT === 'sqlite', }); const moment = require('moment'); const crypto = require('crypto'); @@ -51,11 +52,10 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = - randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase', + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(Util.uploadPath, filename)); @@ -71,7 +71,7 @@ class Util { const retry = async (i = 0) => { const identifier = randomstring.generate({ length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), - capitalization: 'lowercase' + capitalization: 'lowercase', }); const exists = await db .table('links') @@ -138,7 +138,9 @@ class Util { .table('files') .where({ id: fileAlbum.fileId }) .first(); + if (!file) continue; + await this.deleteFile(file.name, true); } } catch (error) { @@ -211,13 +213,15 @@ class Util { '..', process.env.UPLOAD_FOLDER, 'zips', - `${album.userId}-${album.id}.zip` - ) + `${album.userId}-${album.id}.zip`, + ), ); } catch (error) { log.error(error); } } + + static generateThumbnails = ThumbUtil.generateThumbnails; } module.exports = Util; -- cgit v1.2.3 From ad852de51a0d2dd5d29c08838d5a430c58849e74 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 04:00:12 +0300 Subject: chore: linter the entire project using the new rules --- src/api/utils/Util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index ab59c95..905948a 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -52,7 +52,7 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ + const filename = randomstring.generate({ length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase', }) + path.extname(name).toLowerCase(); @@ -81,7 +81,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < 5) return retry(++i); + if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for album'); return null; }; -- cgit v1.2.3