From a42cf4400eb00d3e476e29223d9c3587d61a105a Mon Sep 17 00:00:00 2001 From: Pitu <7425261+Pitu@users.noreply.github.com> Date: Sun, 16 Sep 2018 00:55:41 -0300 Subject: Utils --- src/api/utils/Util.js | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 src/api/utils/Util.js (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js new file mode 100644 index 0000000..46f56d5 --- /dev/null +++ b/src/api/utils/Util.js @@ -0,0 +1,181 @@ +const config = require('../../../config'); +const jetpack = require('fs-jetpack'); +const randomstring = require('randomstring'); +const path = require('path'); +const JWT = require('jsonwebtoken'); +const db = require('knex')(config.server.database); +const moment = require('moment'); +const log = require('../utils/Log'); +const crypto = require('crypto'); +const sharp = require('sharp'); +const ffmpeg = require('fluent-ffmpeg'); + +const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; +const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; + +class Util { + static isExtensionBlocked(extension) { + return config.uploads.blockedExtensions.includes(extension); + } + + static generateThumbnails(filename) { + const ext = path.extname(filename).toLowerCase(); + const output = `${filename.slice(0, -ext.length)}.png`; + if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); + if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); + return null; + } + + /* + static async removeExif(filename) { + This needs more testing. + Even though the exif data seems to be stripped, no other online service + is recognizing the file as an image file. + + const ExifTransformer = require('exif-be-gone'); + const toStream = require('buffer-to-stream'); + + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); + const writer = jetpack.createWriteStream(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, `${filename}.noexif`)); + toStream(file).pipe(new ExifTransformer()).pipe(writer); + } + */ + + static async generateThumbnailForImage(filename, output) { + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); + await sharp(file) + .resize(64, 64) + .toFormat('png') + .toFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square', output)); + await sharp(file) + .resize(225, null) + .toFormat('png') + .toFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', output)); + } + + static generateThumbnailForVideo(filename) { + ffmpeg(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square'), + size: '64x64' + }) + .on('error', error => log.error(error.message)); + ffmpeg(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs'), + size: '150x?' + }) + .on('error', error => log.error(error.message)); + } + + static getFileThumbnail(filename) { + const ext = path.extname(filename).toLowerCase(); + if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; + return `${filename.slice(0, -ext.length)}.png`; + } + + static constructFilePublicLink(file) { + file.url = `${config.filesServeLocation}/${file.name}`; + const thumb = this.getFileThumbnail(file.name); + if (thumb) { + file.thumb = `${config.filesServeLocation}/thumbs/${thumb}`; + file.thumbSquare = `${config.filesServeLocation}/thumbs/square/${thumb}`; + } + return file; + } + + static getUniqueFilename(name) { + const retry = (i = 0) => { + const filename = randomstring.generate({ + length: config.uploads.generatedFilenameLength, + capitalization: 'lowercase' + }) + path.extname(name); + const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); + if (!exists) return filename; + if (i < config.uploads.retryFilenameTimes) return retry(i++); + return null; + }; + return retry(); + } + + static getUniqueAlbumIdentifier() { + const retry = async (i = 0) => { + const identifier = randomstring.generate({ + length: config.uploads.generatedAlbumLinkLength, + capitalization: 'lowercase' + }); + const exists = await db.table('links').where({ identifier }).first(); + if (!exists) return identifier; + if (i < config.uploads.retryAlbumLinkTimes) return retry(i++); + return null; + }; + return retry(); + } + + static async getFileHash(filename) { + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); + if (!file) { + log.error(`There was an error reading the file < ${filename} > for hashing`); + return null; + } + + const hash = crypto.createHash('md5'); + hash.update(file, 'utf8'); + return hash.digest('hex'); + } + + static getFilenameFromPath(fullPath) { + return fullPath.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape + } + + static async deleteFile(filename, deleteFromDB = false) { + try { + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); + if (deleteFromDB) { + await db.table('files').where('name', filename).delete(); + } + } catch (error) { + log.error(`There was an error removing the file < ${filename} >`); + log.error(error); + } + } + + static async deleteAllFilesFromAlbum(id) { + try { + const files = await db.table('files').where({ albumId: id }); + for (const file of files) { + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, file)); + } + await db.table('files').where({ albumId: id }).delete(); + } catch (error) { + log.error(error); + } + } + + static isAuthorized(req) { + if (!req.headers.authorization) return false; + const token = req.headers.authorization.split(' ')[1]; + if (!token) return false; + + return JWT.verify(token, config.server.secret, async (error, decoded) => { + if (error) { + log.error(error); + return false; + } + const id = decoded ? decoded.sub : ''; + const iat = decoded ? decoded.iat : ''; + + const user = await db.table('users').where({ id }).first(); + if (!user || !user.enabled) return false; + if (iat && iat < moment(user.passwordEditedAt).format('x')) return false; + + return user; + }); + } +} + +module.exports = Util; -- cgit v1.2.3 From d777439c7b9498f1db2d42595f1b793d266dfc89 Mon Sep 17 00:00:00 2001 From: Pitu <7425261+Pitu@users.noreply.github.com> Date: Sun, 16 Sep 2018 17:53:26 -0300 Subject: Flawed logic on the async retry --- src/api/utils/Util.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 46f56d5..d8ae735 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -96,7 +96,7 @@ class Util { }) + path.extname(name); const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); if (!exists) return filename; - if (i < config.uploads.retryFilenameTimes) return retry(i++); + if (i < config.uploads.retryFilenameTimes) return retry(i + 1); return null; }; return retry(); @@ -110,7 +110,11 @@ class Util { }); const exists = await db.table('links').where({ identifier }).first(); if (!exists) return identifier; - if (i < config.uploads.retryAlbumLinkTimes) return retry(i++); + /* + It's funny but if you do i++ the asignment never gets done resulting in an infinite loop + */ + if (i < config.uploads.retryAlbumLinkTimes) return retry(i + 1); + log.error('Couldnt allocate identifier for album'); return null; }; return retry(); -- cgit v1.2.3 From c2c6e99878853fafdbd5e708c3163921f8529ae1 Mon Sep 17 00:00:00 2001 From: Pitu <7425261+Pitu@users.noreply.github.com> Date: Mon, 17 Sep 2018 04:38:25 -0300 Subject: Public albums wooo! --- src/api/utils/Util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index d8ae735..e0ad031 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -105,7 +105,7 @@ class Util { static getUniqueAlbumIdentifier() { const retry = async (i = 0) => { const identifier = randomstring.generate({ - length: config.uploads.generatedAlbumLinkLength, + length: config.albums.generatedAlbumLinkLength, capitalization: 'lowercase' }); const exists = await db.table('links').where({ identifier }).first(); @@ -113,7 +113,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < config.uploads.retryAlbumLinkTimes) return retry(i + 1); + if (i < config.albums.retryAlbumLinkTimes) return retry(i + 1); log.error('Couldnt allocate identifier for album'); return null; }; -- cgit v1.2.3 From 1fe6f579f97fdf18200014d28edff1977b692cac Mon Sep 17 00:00:00 2001 From: Pitu <7425261+Pitu@users.noreply.github.com> Date: Tue, 18 Sep 2018 01:44:58 -0300 Subject: Delete thumbs when deleting a file --- src/api/utils/Util.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index e0ad031..617b38f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -137,8 +137,11 @@ class Util { } static async deleteFile(filename, deleteFromDB = false) { + const thumbName = this.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', thumbName)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square', thumbName)); if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } -- cgit v1.2.3 From 4b2b02110b457d8ebeee78e1bdf99eb0660d0626 Mon Sep 17 00:00:00 2001 From: Pitu <7425261+Pitu@users.noreply.github.com> Date: Tue, 18 Sep 2018 03:34:00 -0300 Subject: We can now download albums yayyyy --- src/api/utils/Util.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 617b38f..52cfb03 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -9,6 +9,7 @@ const log = require('../utils/Log'); const crypto = require('crypto'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); +const Zip = require('adm-zip'); const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; @@ -183,6 +184,18 @@ class Util { return user; }); } + + static createZip(files, album) { + try { + const zip = new Zip(); + for (const file of files) { + zip.addLocalFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, file)); + } + zip.writeZip(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'zips', `${album.userId}-${album.id}.zip`)); + } catch (error) { + log.error(error); + } + } } module.exports = Util; -- cgit v1.2.3 From 89a271818ed25b0a17a17dd1d6804e34d1f2ec0f Mon Sep 17 00:00:00 2001 From: Pitu Date: Tue, 19 Feb 2019 23:52:24 +0900 Subject: Switch config to .env --- src/api/utils/Util.js | 66 +++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 28 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 52cfb03..79e933f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -1,9 +1,17 @@ -const config = require('../../../config'); +// const config = require('../../../config'); const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); const JWT = require('jsonwebtoken'); -const db = require('knex')(config.server.database); +const db = require('knex')({ + client: process.env.DB_CLIENT, + connection: { + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASS, + database: process.env.DB_DATABASE + } +}); const moment = require('moment'); const log = require('../utils/Log'); const crypto = require('crypto'); @@ -13,10 +21,11 @@ const Zip = require('adm-zip'); const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; +const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { static isExtensionBlocked(extension) { - return config.uploads.blockedExtensions.includes(extension); + return blockedExtensions.includes(extension); } static generateThumbnails(filename) { @@ -36,38 +45,38 @@ class Util { const ExifTransformer = require('exif-be-gone'); const toStream = require('buffer-to-stream'); - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); - const writer = jetpack.createWriteStream(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, `${filename}.noexif`)); + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); + const writer = jetpack.createWriteStream(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, `${filename}.noexif`)); toStream(file).pipe(new ExifTransformer()).pipe(writer); } */ static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); await sharp(file) .resize(64, 64) .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square', output)); + .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); await sharp(file) .resize(225, null) .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', output)); + .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); } static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)) + ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) .thumbnail({ timestamps: [0], filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square'), + folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), size: '64x64' }) .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)) + ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) .thumbnail({ timestamps: [0], filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs'), + folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), size: '150x?' }) .on('error', error => log.error(error.message)); @@ -80,11 +89,11 @@ class Util { } static constructFilePublicLink(file) { - file.url = `${config.filesServeLocation}/${file.name}`; + file.url = `${process.env.DOMAIN}/${file.name}`; const thumb = this.getFileThumbnail(file.name); if (thumb) { - file.thumb = `${config.filesServeLocation}/thumbs/${thumb}`; - file.thumbSquare = `${config.filesServeLocation}/thumbs/square/${thumb}`; + file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; + file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; } return file; } @@ -92,12 +101,13 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { const filename = randomstring.generate({ - length: config.uploads.generatedFilenameLength, + length: process.env.GENERATED_FILENAME_LENGTH, capitalization: 'lowercase' }) + path.extname(name); - const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); + const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); if (!exists) return filename; - if (i < config.uploads.retryFilenameTimes) return retry(i + 1); + if (i < 5) return retry(i + 1); + log.error('Couldnt allocate identifier for file'); return null; }; return retry(); @@ -106,7 +116,7 @@ class Util { static getUniqueAlbumIdentifier() { const retry = async (i = 0) => { const identifier = randomstring.generate({ - length: config.albums.generatedAlbumLinkLength, + length: process.env.GENERATED_ALBUM_LENGTH, capitalization: 'lowercase' }); const exists = await db.table('links').where({ identifier }).first(); @@ -114,7 +124,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < config.albums.retryAlbumLinkTimes) return retry(i + 1); + if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for album'); return null; }; @@ -122,7 +132,7 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename), 'buffer'); + const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -140,9 +150,9 @@ class Util { static async deleteFile(filename, deleteFromDB = false) { const thumbName = this.getFileThumbnail(filename); try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'thumbs', 'square', thumbName)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } @@ -156,7 +166,7 @@ class Util { try { const files = await db.table('files').where({ albumId: id }); for (const file of files) { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, file)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); } await db.table('files').where({ albumId: id }).delete(); } catch (error) { @@ -169,7 +179,7 @@ class Util { const token = req.headers.authorization.split(' ')[1]; if (!token) return false; - return JWT.verify(token, config.server.secret, async (error, decoded) => { + return JWT.verify(token, process.env.SECRET, async (error, decoded) => { if (error) { log.error(error); return false; @@ -189,9 +199,9 @@ class Util { try { const zip = new Zip(); for (const file of files) { - zip.addLocalFile(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, file)); + zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); } - zip.writeZip(path.join(__dirname, '..', '..', '..', config.uploads.uploadFolder, 'zips', `${album.userId}-${album.id}.zip`)); + zip.writeZip(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'zips', `${album.userId}-${album.id}.zip`)); } catch (error) { log.error(error); } -- cgit v1.2.3 From 25c5a06ec3e363f5b98607949b76b8b395e4c962 Mon Sep 17 00:00:00 2001 From: Pitu Date: Thu, 21 Feb 2019 23:05:56 +0900 Subject: derp --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 79e933f..46f7d2e 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -8,7 +8,7 @@ const db = require('knex')({ connection: { host: process.env.DB_HOST, user: process.env.DB_USER, - password: process.env.DB_PASS, + password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE } }); -- cgit v1.2.3 From c7a4a39de4e6113e88f07fefb3668e9fd3b1372a Mon Sep 17 00:00:00 2001 From: Pitu Date: Fri, 22 Feb 2019 00:00:07 +0900 Subject: Add support for sqlite --- src/api/utils/Util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 46f7d2e..0169a8f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -9,7 +9,8 @@ const db = require('knex')({ host: process.env.DB_HOST, user: process.env.DB_USER, password: process.env.DB_PASSWORD, - database: process.env.DB_DATABASE + database: process.env.DB_DATABASE, + filename: '../../../database.sqlite' } }); const moment = require('moment'); -- cgit v1.2.3 From df90d3157a6fac0ef8880c17b06d5389f31265c3 Mon Sep 17 00:00:00 2001 From: Kana Date: Fri, 22 Feb 2019 15:07:37 +0900 Subject: Update Util.js --- src/api/utils/Util.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 0169a8f..f99d8e0 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -1,4 +1,3 @@ -// const config = require('../../../config'); const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); @@ -10,7 +9,7 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: '../../../database.sqlite' + filename: path.join(__dirname, '..', '..', '..', 'database.sqlite') } }); const moment = require('moment'); -- cgit v1.2.3 From fc95cb7b0f047806937c25f0fc1104c72b0a32cb Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 23 Feb 2019 00:45:45 +0900 Subject: Better DB handling and stuff --- src/api/utils/Util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index f99d8e0..26edf4b 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -10,7 +10,8 @@ const db = require('knex')({ password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, filename: path.join(__dirname, '..', '..', '..', 'database.sqlite') - } + }, + useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false }); const moment = require('moment'); const log = require('../utils/Log'); -- cgit v1.2.3 From 9f5a3d15f55fea03052627f3bd4d97a4284cdf7c Mon Sep 17 00:00:00 2001 From: Pitu Date: Thu, 28 Feb 2019 23:51:59 +0900 Subject: Purge user's files --- src/api/utils/Util.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 26edf4b..9e9753c 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -90,6 +90,10 @@ class Util { } static constructFilePublicLink(file) { + /* + TODO: This wont work without a reverse proxy serving both + the site and the API under the same domain. Pls fix. + */ file.url = `${process.env.DOMAIN}/${file.name}`; const thumb = this.getFileThumbnail(file.name); if (thumb) { @@ -175,6 +179,18 @@ class Util { } } + static async deleteAllFilesFromUser(id) { + try { + const files = await db.table('files').where({ userId: id }); + for (const file of files) { + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + } + await db.table('files').where({ userId: id }).delete(); + } catch (error) { + log.error(error); + } + } + static isAuthorized(req) { if (!req.headers.authorization) return false; const token = req.headers.authorization.split(' ')[1]; -- cgit v1.2.3 From 73d85e8c7938e1db30da3cc4354b143d4a078473 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 2 Mar 2019 02:08:11 +0900 Subject: Enviroment variables parsing fix --- src/api/utils/Util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 9e9753c..1242a5a 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -106,7 +106,7 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { const filename = randomstring.generate({ - length: process.env.GENERATED_FILENAME_LENGTH, + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase' }) + path.extname(name); const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); @@ -121,7 +121,7 @@ class Util { static getUniqueAlbumIdentifier() { const retry = async (i = 0) => { const identifier = randomstring.generate({ - length: process.env.GENERATED_ALBUM_LENGTH, + length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), capitalization: 'lowercase' }); const exists = await db.table('links').where({ identifier }).first(); -- cgit v1.2.3 From ce76a8ec7ba2084ab75fb901933b3d4d84505032 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 2 Mar 2019 22:07:24 +0900 Subject: Be able to add a file to multiple albums --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 1242a5a..a514e20 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -108,7 +108,7 @@ class Util { const filename = randomstring.generate({ length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase' - }) + path.extname(name); + }) + path.extname(name).toLowerCase(); const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); if (!exists) return filename; if (i < 5) return retry(i + 1); -- cgit v1.2.3 From 4277db90f670f72cd3729ad88b91a4d4993de1cf Mon Sep 17 00:00:00 2001 From: Pitu Date: Tue, 12 Mar 2019 06:18:32 +0000 Subject: Possible fix for files not being purged --- src/api/utils/Util.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index a514e20..a8e1754 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -109,6 +109,8 @@ class Util { length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase' }) + path.extname(name).toLowerCase(); + + // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); if (!exists) return filename; if (i < 5) return retry(i + 1); @@ -171,9 +173,8 @@ class Util { try { const files = await db.table('files').where({ albumId: id }); for (const file of files) { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + await this.deleteFile(file.name, true) } - await db.table('files').where({ albumId: id }).delete(); } catch (error) { log.error(error); } @@ -183,9 +184,8 @@ class Util { try { const files = await db.table('files').where({ userId: id }); for (const file of files) { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + await this.deleteFile(file.name, true) } - await db.table('files').where({ userId: id }).delete(); } catch (error) { log.error(error); } -- cgit v1.2.3 From af9d752eaf80a7ee2eef6ab3fafd97e4004572ed Mon Sep 17 00:00:00 2001 From: Pitu Date: Thu, 14 Mar 2019 23:13:51 +0900 Subject: Add uuid package --- src/api/utils/Util.js | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index a8e1754..1776f3e 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -19,12 +19,17 @@ const crypto = require('crypto'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); const Zip = require('adm-zip'); +const uuidv4 = require('uuid/v4'); const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { + static uuid() { + return uuidv4(); + } + static isExtensionBlocked(extension) { return blockedExtensions.includes(extension); } @@ -171,9 +176,13 @@ class Util { static async deleteAllFilesFromAlbum(id) { try { - const files = await db.table('files').where({ albumId: id }); - for (const file of files) { - await this.deleteFile(file.name, true) + const fileAlbums = await db.table('albumsFiles').where({ albumId: id }); + for (const fileAlbum of fileAlbums) { + const file = await db.table('files') + .where({ id: fileAlbum.fileId }) + .first(); + if (!file) continue; + await this.deleteFile(file.name, true); } } catch (error) { log.error(error); @@ -184,7 +193,22 @@ class Util { try { const files = await db.table('files').where({ userId: id }); for (const file of files) { - await this.deleteFile(file.name, true) + await this.deleteFile(file.name, true); + } + } catch (error) { + log.error(error); + } + } + + static async deleteAllFilesFromTag(id) { + try { + const fileTags = await db.table('fileTags').where({ tagId: id }); + for (const fileTag of fileTags) { + const file = await db.table('files') + .where({ id: fileTag.fileId }) + .first(); + if (!file) continue; + await this.deleteFile(file.name, true); } } catch (error) { log.error(error); -- cgit v1.2.3 From 579e1e754ab59a69925b5114641174aa70d18555 Mon Sep 17 00:00:00 2001 From: Pitu Date: Tue, 1 Oct 2019 14:11:16 -0300 Subject: feature: uploader with chunks support --- src/api/utils/Util.js | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 1776f3e..d7c9623 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -155,6 +155,11 @@ class Util { return hash.digest('hex'); } + static generateFileHash(data) { + const hash = crypto.createHash('sha1').update(data).digest('hex'); + return hash; + } + static getFilenameFromPath(fullPath) { return fullPath.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape } -- cgit v1.2.3 From c121bd42f38cc3bd47b68efd8cf70da158cbdf8d Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 12 Oct 2019 14:37:09 +0900 Subject: chore: upload fixes --- src/api/utils/Util.js | 1 + 1 file changed, 1 insertion(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index d7c9623..0251cc0 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -89,6 +89,7 @@ class Util { } static getFileThumbnail(filename) { + if (!filename) return null; const ext = path.extname(filename).toLowerCase(); if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; return `${filename.slice(0, -ext.length)}.png`; -- cgit v1.2.3 From 459ab5433b9c3b60f43f9b2293189be8e29e0e84 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 12 Oct 2019 14:58:58 +0900 Subject: chore: remove exif strip support. After some thought, modifying uploaded files is not something I want to support. --- src/api/utils/Util.js | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 0251cc0..7ee32b6 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -42,21 +42,6 @@ class Util { return null; } - /* - static async removeExif(filename) { - This needs more testing. - Even though the exif data seems to be stripped, no other online service - is recognizing the file as an image file. - - const ExifTransformer = require('exif-be-gone'); - const toStream = require('buffer-to-stream'); - - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); - const writer = jetpack.createWriteStream(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, `${filename}.noexif`)); - toStream(file).pipe(new ExifTransformer()).pipe(writer); - } - */ - static async generateThumbnailForImage(filename, output) { const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); await sharp(file) -- cgit v1.2.3 From a639b85734e4ab3f504214c61807d4ac7b0882c7 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 9 May 2020 23:56:35 +0900 Subject: Fix: consistent hash of uploads --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 7ee32b6..b8d960d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -142,7 +142,7 @@ class Util { } static generateFileHash(data) { - const hash = crypto.createHash('sha1').update(data).digest('hex'); + const hash = crypto.createHash('md5').update(data).digest('hex'); return hash; } -- cgit v1.2.3 From 520062508ccad88d49229e603fc4d2c0c0a118d3 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Mon, 29 Jun 2020 22:18:42 +0300 Subject: feat: backend pagination serverLoad++; userRamUsage--; --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index b8d960d..c37297a 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -122,7 +122,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < 5) return retry(i + 1); + if (i < 5) return retry(++i); log.error('Couldnt allocate identifier for album'); return null; }; -- cgit v1.2.3 From 3e1677c18a2c423f0088bf107528938d77471259 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 1 Jul 2020 20:13:34 +0300 Subject: fix: remove .bmp from the imageExtensions because sharp doesn't support it Trying to generate a thumb would throw a not supported exception, which would crash newer node.js versions because the process (on older version, it would only throw a UnhandledPromiseRejectionWarning) --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index c37297a..07c295d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -21,7 +21,7 @@ const ffmpeg = require('fluent-ffmpeg'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); -const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; +const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); -- cgit v1.2.3 From 42f1a1003a299ca4e571d503d7200b76ff41e752 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:11:51 +0300 Subject: feat: externalize thumb generation function for easier testing --- src/api/utils/Util.js | 118 ++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 67 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 07c295d..8c9428d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -14,15 +14,13 @@ const db = require('knex')({ useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false }); const moment = require('moment'); -const log = require('../utils/Log'); const crypto = require('crypto'); -const sharp = require('sharp'); -const ffmpeg = require('fluent-ffmpeg'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); -const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; +const log = require('./Log'); +const ThumbUtil = require('./ThumbUtil'); + const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { @@ -34,59 +32,13 @@ class Util { return blockedExtensions.includes(extension); } - static generateThumbnails(filename) { - const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; - if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); - return null; - } - - static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); - await sharp(file) - .resize(64, 64) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); - await sharp(file) - .resize(225, null) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); - } - - static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), - size: '64x64' - }) - .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), - size: '150x?' - }) - .on('error', error => log.error(error.message)); - } - - static getFileThumbnail(filename) { - if (!filename) return null; - const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; - } - static constructFilePublicLink(file) { /* TODO: This wont work without a reverse proxy serving both the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = this.getFileThumbnail(file.name); + const thumb = ThumbUtil.getFileThumbnail(file.name); if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; @@ -96,10 +48,11 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = + randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase' + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); @@ -117,7 +70,10 @@ class Util { length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), capitalization: 'lowercase' }); - const exists = await db.table('links').where({ identifier }).first(); + const exists = await db + .table('links') + .where({ identifier }) + .first(); if (!exists) return identifier; /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop @@ -130,7 +86,10 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); + const file = await jetpack.readAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), + 'buffer' + ); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -142,7 +101,10 @@ class Util { } static generateFileHash(data) { - const hash = crypto.createHash('md5').update(data).digest('hex'); + const hash = crypto + .createHash('md5') + .update(data) + .digest('hex'); return hash; } @@ -151,13 +113,20 @@ class Util { } static async deleteFile(filename, deleteFromDB = false) { - const thumbName = this.getFileThumbnail(filename); + const thumbName = ThumbUtil.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) + ); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) + ); if (deleteFromDB) { - await db.table('files').where('name', filename).delete(); + await db + .table('files') + .where('name', filename) + .delete(); } } catch (error) { log.error(`There was an error removing the file < ${filename} >`); @@ -169,7 +138,8 @@ class Util { try { const fileAlbums = await db.table('albumsFiles').where({ albumId: id }); for (const fileAlbum of fileAlbums) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileAlbum.fileId }) .first(); if (!file) continue; @@ -195,7 +165,8 @@ class Util { try { const fileTags = await db.table('fileTags').where({ tagId: id }); for (const fileTag of fileTags) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileTag.fileId }) .first(); if (!file) continue; @@ -219,7 +190,10 @@ class Util { const id = decoded ? decoded.sub : ''; const iat = decoded ? decoded.iat : ''; - const user = await db.table('users').where({ id }).first(); + const user = await db + .table('users') + .where({ id }) + .first(); if (!user || !user.enabled) return false; if (iat && iat < moment(user.passwordEditedAt).format('x')) return false; @@ -233,7 +207,17 @@ class Util { for (const file of files) { zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); } - zip.writeZip(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'zips', `${album.userId}-${album.id}.zip`)); + zip.writeZip( + path.join( + __dirname, + '..', + '..', + '..', + process.env.UPLOAD_FOLDER, + 'zips', + `${album.userId}-${album.id}.zip` + ) + ); } catch (error) { log.error(error); } -- cgit v1.2.3 From a790d7749e04d71df1613e6a02258982683aa290 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:42:20 +0300 Subject: feat: add experimental meaningful preview extraction from videos For now, it sitll requires gifski. It could be rewritten to use webp instead of gifs, because that is a lot faster, uses less space and we could use ffmpeg for it. --- src/api/utils/Util.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 8c9428d..c997581 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -24,6 +24,8 @@ const ThumbUtil = require('./ThumbUtil'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { + static uploadPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER); + static uuid() { return uuidv4(); } @@ -55,7 +57,7 @@ class Util { }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem - const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + const exists = jetpack.exists(path.join(Util.uploadPath, filename)); if (!exists) return filename; if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for file'); @@ -86,10 +88,7 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), - 'buffer' - ); + const file = await jetpack.readAsync(path.join(Util.uploadPath, filename), 'buffer'); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -115,13 +114,9 @@ class Util { static async deleteFile(filename, deleteFromDB = false) { const thumbName = ThumbUtil.getFileThumbnail(filename); try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) - ); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) - ); + await jetpack.removeAsync(path.join(Util.uploadPath, filename)); + await ThumbUtil.removeThumbs(thumbName); + if (deleteFromDB) { await db .table('files') @@ -205,7 +200,7 @@ class Util { try { const zip = new Zip(); for (const file of files) { - zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + zip.addLocalFile(path.join(Util.uploadPath, file)); } zip.writeZip( path.join( -- cgit v1.2.3 From 1e1f3fbb27976a34f53a4e8d250da34dad4e6c20 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sat, 4 Jul 2020 23:18:51 +0300 Subject: feat: experimental videos in grid --- src/api/utils/Util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index c997581..496ea18 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -40,10 +40,11 @@ class Util { the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = ThumbUtil.getFileThumbnail(file.name); + const { thumb, preview } = ThumbUtil.getFileThumbnail(file.name) || {}; if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; + file.preview = preview && `${process.env.DOMAIN}/thumbs/preview/${preview}`; } return file; } -- cgit v1.2.3 From eccbb1ca93f1b86e9bc93dcbc1ec0ee9b168d949 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 02:32:12 +0300 Subject: fix: errors in Util caused by separating into different classes improperly --- src/api/utils/Util.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 496ea18..ab59c95 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); @@ -9,9 +10,9 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: path.join(__dirname, '..', '..', '..', 'database.sqlite') + filename: path.join(__dirname, '../../../database.sqlite'), }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false + useNullAsDefault: process.env.DB_CLIENT === 'sqlite', }); const moment = require('moment'); const crypto = require('crypto'); @@ -51,11 +52,10 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = - randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase', + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(Util.uploadPath, filename)); @@ -71,7 +71,7 @@ class Util { const retry = async (i = 0) => { const identifier = randomstring.generate({ length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), - capitalization: 'lowercase' + capitalization: 'lowercase', }); const exists = await db .table('links') @@ -138,7 +138,9 @@ class Util { .table('files') .where({ id: fileAlbum.fileId }) .first(); + if (!file) continue; + await this.deleteFile(file.name, true); } } catch (error) { @@ -211,13 +213,15 @@ class Util { '..', process.env.UPLOAD_FOLDER, 'zips', - `${album.userId}-${album.id}.zip` - ) + `${album.userId}-${album.id}.zip`, + ), ); } catch (error) { log.error(error); } } + + static generateThumbnails = ThumbUtil.generateThumbnails; } module.exports = Util; -- cgit v1.2.3 From ad852de51a0d2dd5d29c08838d5a430c58849e74 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 04:00:12 +0300 Subject: chore: linter the entire project using the new rules --- src/api/utils/Util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index ab59c95..905948a 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -52,7 +52,7 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ + const filename = randomstring.generate({ length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase', }) + path.extname(name).toLowerCase(); @@ -81,7 +81,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < 5) return retry(++i); + if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for album'); return null; }; -- cgit v1.2.3 From 4dafc79cb74d901bb9454f78277298f020543bb5 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 02:55:05 +0900 Subject: fix authorization --- src/api/utils/Util.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index b8d960d..80bffd5 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -206,7 +206,15 @@ class Util { } } - static isAuthorized(req) { + static async isAuthorized(req) { + if (req.headers.token) { + if (!this.options.canApiKey) return false; + const user = await db.table('users').where({ apiKey: req.headers.token }).first(); + if (!user) return false; + if (!user.enabled) return false; + return true; + } + if (!req.headers.authorization) return false; const token = req.headers.authorization.split(' ')[1]; if (!token) return false; -- cgit v1.2.3 From 5f58431409e1a4e875cd8121cfe9dc47cfecc65e Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 02:57:24 +0900 Subject: Fix authorization --- src/api/utils/Util.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 80bffd5..7f6dd22 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -208,10 +208,8 @@ class Util { static async isAuthorized(req) { if (req.headers.token) { - if (!this.options.canApiKey) return false; const user = await db.table('users').where({ apiKey: req.headers.token }).first(); - if (!user) return false; - if (!user.enabled) return false; + if (!user || !user.enabled) return false; return true; } -- cgit v1.2.3 From 407fb8bcc31cd69394a2444db53b710cc2dc4d55 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:05:12 +0900 Subject: Fix authorization --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 7f6dd22..91ab663 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -210,7 +210,7 @@ class Util { if (req.headers.token) { const user = await db.table('users').where({ apiKey: req.headers.token }).first(); if (!user || !user.enabled) return false; - return true; + return user; } if (!req.headers.authorization) return false; -- cgit v1.2.3 From 6dd7500084bf7306f66e8f65367b90f1049e3f15 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:35:08 +0900 Subject: Fix deleting files without thumb --- src/api/utils/Util.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 91ab663..4674dde 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -154,8 +154,6 @@ class Util { const thumbName = this.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } @@ -163,6 +161,14 @@ class Util { log.error(`There was an error removing the file < ${filename} >`); log.error(error); } + + try { + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); + await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); + } catch (error) { + log.error(`There was an error removing the thumbs for file < ${filename} >`); + log.error(error); + } } static async deleteAllFilesFromAlbum(id) { -- cgit v1.2.3 From b70a75da1af00e932319b3ba24ccde860a6c7f48 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 03:37:52 +0900 Subject: Fix for real --- src/api/utils/Util.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 4674dde..885228f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -154,6 +154,13 @@ class Util { const thumbName = this.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + if (thumbName) { + const thumb = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName); + const thumbSquare = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName); + if (await jetpack.existsAsync(thumb)) jetpack.removeAsync(thumb); + if (await jetpack.existsAsync(thumbSquare)) jetpack.removeAsync(thumbSquare); + } + if (deleteFromDB) { await db.table('files').where('name', filename).delete(); } @@ -161,14 +168,6 @@ class Util { log.error(`There was an error removing the file < ${filename} >`); log.error(error); } - - try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); - } catch (error) { - log.error(`There was an error removing the thumbs for file < ${filename} >`); - log.error(error); - } } static async deleteAllFilesFromAlbum(id) { -- cgit v1.2.3 From d644b21d431c03263aa7191fe54a984aac96f979 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sat, 18 Jul 2020 05:50:47 +0900 Subject: Make thumbnails webp (bye bye safari) --- src/api/utils/Util.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 885228f..a4af81e 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -36,7 +36,7 @@ class Util { static generateThumbnails(filename) { const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; + const output = `${filename.slice(0, -ext.length)}.webp`; if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); return null; @@ -46,11 +46,11 @@ class Util { const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); await sharp(file) .resize(64, 64) - .toFormat('png') + .toFormat('webp') .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); await sharp(file) .resize(225, null) - .toFormat('png') + .toFormat('webp') .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); } @@ -76,8 +76,9 @@ class Util { static getFileThumbnail(filename) { if (!filename) return null; const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; + const extension = imageExtensions.includes(ext) ? 'webp' : videoExtensions.includes(ext) ? 'png' : null; + if (!extension) return null; + return `${filename.slice(0, -ext.length)}.${extension}`; } static constructFilePublicLink(file) { -- cgit v1.2.3 From 90001c2df56d58e69fd199a518ae7f3e4ed327fc Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 24 Dec 2020 10:40:50 +0200 Subject: chore: remove trailing commas --- src/api/utils/Util.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index ee4c748..4279b6f 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -10,9 +10,9 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: path.join(__dirname, '../../../database.sqlite'), + filename: path.join(__dirname, '../../../database.sqlite') }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite', + useNullAsDefault: process.env.DB_CLIENT === 'sqlite' }); const moment = require('moment'); const crypto = require('crypto'); @@ -25,7 +25,7 @@ const ThumbUtil = require('./ThumbUtil'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { - static uploadPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER); + static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER); static uuid() { return uuidv4(); @@ -54,7 +54,7 @@ class Util { const retry = (i = 0) => { const filename = randomstring.generate({ length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase', + capitalization: 'lowercase' }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem @@ -71,7 +71,7 @@ class Util { const retry = async (i = 0) => { const identifier = randomstring.generate({ length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), - capitalization: 'lowercase', + capitalization: 'lowercase' }); const exists = await db .table('links') @@ -214,13 +214,11 @@ class Util { zip.writeZip( path.join( __dirname, - '..', - '..', - '..', + '../../../', process.env.UPLOAD_FOLDER, 'zips', - `${album.userId}-${album.id}.zip`, - ), + `${album.userId}-${album.id}.zip` + ) ); } catch (error) { log.error(error); -- cgit v1.2.3 From f73cde6bb501d72e46f0aadf95e6809e9d265e5b Mon Sep 17 00:00:00 2001 From: Pitu Date: Fri, 25 Dec 2020 03:58:19 +0900 Subject: Chore: Move database to a subfolder for docker purposes --- src/api/utils/Util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 4279b6f..35d726e 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -10,7 +10,7 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: path.join(__dirname, '../../../database.sqlite') + filename: path.join(__dirname, '../../../database/database.sqlite') }, useNullAsDefault: process.env.DB_CLIENT === 'sqlite' }); -- cgit v1.2.3 From e97fee48441717f3b508ac855339d0fb8210be53 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sun, 27 Dec 2020 04:27:56 +0900 Subject: Fixes chunked uploads not being saved to albums or thumbnails --- src/api/utils/Util.js | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 35d726e..905f217 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -226,6 +226,60 @@ class Util { } static generateThumbnails = ThumbUtil.generateThumbnails; + static async saveFileToDatabase(req, res, user, db, file, originalFile) { + /* + Save the upload information to the database + */ + const now = moment.utc().toDate(); + let insertedId = null; + try { + /* + This is so fucking dumb + */ + if (process.env.DB_CLIENT === 'sqlite3') { + insertedId = await db.table('files').insert({ + userId: user ? user.id : null, + name: file.name, + original: originalFile.originalname, + type: originalFile.mimetype || '', + size: file.size, + hash: file.hash, + ip: req.ip, + createdAt: now, + editedAt: now + }); + } else { + insertedId = await db.table('files').insert({ + userId: user ? user.id : null, + name: file.name, + original: originalFile.originalname, + type: originalFile.mimetype || '', + size: file.size, + hash: file.hash, + ip: req.ip, + createdAt: now, + editedAt: now + }, 'id'); + } + return insertedId; + } catch (error) { + console.error('There was an error saving the file to the database'); + console.error(error); + return null; + } + } + + static async saveFileToAlbum(db, albumId, insertedId) { + if (!albumId) return; + + const now = moment.utc().toDate(); + try { + await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }); + await db.table('albums').where('id', albumId).update('editedAt', now); + } catch (error) { + console.error(error); + } + } } module.exports = Util; -- cgit v1.2.3 From aa7d2453171b3a596a1be6676eaf39cc93fe178f Mon Sep 17 00:00:00 2001 From: Pitu Date: Sun, 27 Dec 2020 04:48:03 +0900 Subject: feat: Add hash checking for chunked uploads --- src/api/utils/Util.js | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 905f217..e52fac2 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -108,6 +108,17 @@ class Util { return hash; } + static async checkIfFileExists(db, user, hash) { + const exists = await db.table('files') + .where(function() { // eslint-disable-line func-names + if (user) this.where('userId', user.id); + else this.whereNull('userId'); + }) + .where({ hash }) + .first(); + return exists; + } + static getFilenameFromPath(fullPath) { return fullPath.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape } -- cgit v1.2.3