From 42f1a1003a299ca4e571d503d7200b76ff41e752 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:11:51 +0300 Subject: feat: externalize thumb generation function for easier testing --- src/api/generateThumbs.js | 57 ++-------------------- src/api/utils/ThumbUtil.js | 60 +++++++++++++++++++++++ src/api/utils/Util.js | 118 ++++++++++++++++++++------------------------- 3 files changed, 115 insertions(+), 120 deletions(-) create mode 100644 src/api/utils/ThumbUtil.js (limited to 'src/api') diff --git a/src/api/generateThumbs.js b/src/api/generateThumbs.js index 8517608..761bd5a 100644 --- a/src/api/generateThumbs.js +++ b/src/api/generateThumbs.js @@ -1,62 +1,13 @@ require('dotenv').config(); -const jetpack = require('fs-jetpack'); -const path = require('path'); -const fs = require('fs'); -const log = require('./utils/Log'); -const sharp = require('sharp'); -const ffmpeg = require('fluent-ffmpeg'); -const imageExtensions = ['.jpg', '.jpeg', '.bmp', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; - -class ThumbGenerator { - static generateThumbnails(filename) { - const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; - if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); - return null; - } - - static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); - await sharp(file) - .resize(64, 64) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); - await sharp(file) - .resize(225, null) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); - } - - static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), - size: '64x64' - }) - .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), - size: '150x?' - }) - .on('error', error => log.error(error.message)); - } -} - +const ThumbUtil = require('./utils/ThumbUtil'); const start = async () => { const files = fs.readdirSync(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER)); for (const fileName of files) { console.log(`Generating thumb for '${fileName}`); - await ThumbGenerator.generateThumbnails(fileName); + await ThumbUtil.generateThumbnails(fileName); } -} +}; -start(); \ No newline at end of file +start(); diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js new file mode 100644 index 0000000..5c96b5c --- /dev/null +++ b/src/api/utils/ThumbUtil.js @@ -0,0 +1,60 @@ +const jetpack = require('fs-jetpack'); +const path = require('path'); +const sharp = require('sharp'); +const ffmpeg = require('fluent-ffmpeg'); + +const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; +const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; + +class ThumbUtil { + static generateThumbnails(filename) { + const ext = path.extname(filename).toLowerCase(); + const output = `${filename.slice(0, -ext.length)}.png`; + if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); + if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); + return null; + } + + static async generateThumbnailForImage(filename, output) { + const file = await jetpack.readAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), + 'buffer' + ); + await sharp(file) + .resize(64, 64) + .toFormat('png') + .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); + await sharp(file) + .resize(225, null) + .toFormat('png') + .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); + } + + static generateThumbnailForVideo(filename) { + ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), + size: '64x64' + }) + .on('error', error => log.error(error.message)); + ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) + .thumbnail({ + timestamps: [0], + filename: '%b.png', + folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), + size: '150x?' + }) + .on('error', error => log.error(error.message)); + } + + static getFileThumbnail(filename) { + if (!filename) return null; + const ext = path.extname(filename).toLowerCase(); + if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; + return `${filename.slice(0, -ext.length)}.png`; + } +} + +module.exports = ThumbUtil; diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 07c295d..8c9428d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -14,15 +14,13 @@ const db = require('knex')({ useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false }); const moment = require('moment'); -const log = require('../utils/Log'); const crypto = require('crypto'); -const sharp = require('sharp'); -const ffmpeg = require('fluent-ffmpeg'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); -const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; +const log = require('./Log'); +const ThumbUtil = require('./ThumbUtil'); + const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { @@ -34,59 +32,13 @@ class Util { return blockedExtensions.includes(extension); } - static generateThumbnails(filename) { - const ext = path.extname(filename).toLowerCase(); - const output = `${filename.slice(0, -ext.length)}.png`; - if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); - return null; - } - - static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); - await sharp(file) - .resize(64, 64) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); - await sharp(file) - .resize(225, null) - .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); - } - - static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), - size: '64x64' - }) - .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) - .thumbnail({ - timestamps: [0], - filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), - size: '150x?' - }) - .on('error', error => log.error(error.message)); - } - - static getFileThumbnail(filename) { - if (!filename) return null; - const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; - } - static constructFilePublicLink(file) { /* TODO: This wont work without a reverse proxy serving both the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = this.getFileThumbnail(file.name); + const thumb = ThumbUtil.getFileThumbnail(file.name); if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; @@ -96,10 +48,11 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = + randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase' + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); @@ -117,7 +70,10 @@ class Util { length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), capitalization: 'lowercase' }); - const exists = await db.table('links').where({ identifier }).first(); + const exists = await db + .table('links') + .where({ identifier }) + .first(); if (!exists) return identifier; /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop @@ -130,7 +86,10 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), 'buffer'); + const file = await jetpack.readAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), + 'buffer' + ); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -142,7 +101,10 @@ class Util { } static generateFileHash(data) { - const hash = crypto.createHash('md5').update(data).digest('hex'); + const hash = crypto + .createHash('md5') + .update(data) + .digest('hex'); return hash; } @@ -151,13 +113,20 @@ class Util { } static async deleteFile(filename, deleteFromDB = false) { - const thumbName = this.getFileThumbnail(filename); + const thumbName = ThumbUtil.getFileThumbnail(filename); try { await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName)); - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName)); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) + ); + await jetpack.removeAsync( + path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) + ); if (deleteFromDB) { - await db.table('files').where('name', filename).delete(); + await db + .table('files') + .where('name', filename) + .delete(); } } catch (error) { log.error(`There was an error removing the file < ${filename} >`); @@ -169,7 +138,8 @@ class Util { try { const fileAlbums = await db.table('albumsFiles').where({ albumId: id }); for (const fileAlbum of fileAlbums) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileAlbum.fileId }) .first(); if (!file) continue; @@ -195,7 +165,8 @@ class Util { try { const fileTags = await db.table('fileTags').where({ tagId: id }); for (const fileTag of fileTags) { - const file = await db.table('files') + const file = await db + .table('files') .where({ id: fileTag.fileId }) .first(); if (!file) continue; @@ -219,7 +190,10 @@ class Util { const id = decoded ? decoded.sub : ''; const iat = decoded ? decoded.iat : ''; - const user = await db.table('users').where({ id }).first(); + const user = await db + .table('users') + .where({ id }) + .first(); if (!user || !user.enabled) return false; if (iat && iat < moment(user.passwordEditedAt).format('x')) return false; @@ -233,7 +207,17 @@ class Util { for (const file of files) { zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); } - zip.writeZip(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'zips', `${album.userId}-${album.id}.zip`)); + zip.writeZip( + path.join( + __dirname, + '..', + '..', + '..', + process.env.UPLOAD_FOLDER, + 'zips', + `${album.userId}-${album.id}.zip` + ) + ); } catch (error) { log.error(error); } -- cgit v1.2.3 From a790d7749e04d71df1613e6a02258982683aa290 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 03:42:20 +0300 Subject: feat: add experimental meaningful preview extraction from videos For now, it sitll requires gifski. It could be rewritten to use webp instead of gifs, because that is a lot faster, uses less space and we could use ffmpeg for it. --- src/api/generateThumbs.js | 3 +++ src/api/utils/ThumbUtil.js | 58 +++++++++++++++++++++++++++++++++------------- src/api/utils/Util.js | 21 +++++++---------- 3 files changed, 53 insertions(+), 29 deletions(-) (limited to 'src/api') diff --git a/src/api/generateThumbs.js b/src/api/generateThumbs.js index 761bd5a..1f2c531 100644 --- a/src/api/generateThumbs.js +++ b/src/api/generateThumbs.js @@ -1,5 +1,8 @@ require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); + const ThumbUtil = require('./utils/ThumbUtil'); const start = async () => { diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index 5c96b5c..e508969 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -2,59 +2,85 @@ const jetpack = require('fs-jetpack'); const path = require('path'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); +const generatePreview = require('ffmpeg-generate-video-preview'); -const imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; -const videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; +const log = require('./Log'); class ThumbUtil { + static imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; + static videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; + + static thumbPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'); + static squareThumbPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'); + static videoPreviewPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'preview'); + static generateThumbnails(filename) { const ext = path.extname(filename).toLowerCase(); const output = `${filename.slice(0, -ext.length)}.png`; - if (imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename); + const previewOutput = `${filename.slice(0, -ext.length)}.gif`; + + if (ThumbUtil.imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); + if (ThumbUtil.videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename, previewOutput); return null; } static async generateThumbnailForImage(filename, output) { - const file = await jetpack.readAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), - 'buffer' - ); + const filePath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename); + + const file = await jetpack.readAsync(filePath, 'buffer'); await sharp(file) .resize(64, 64) .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', output)); + .toFile(path.join(ThumbUtil.squareThumbPath, output)); await sharp(file) .resize(225, null) .toFormat('png') - .toFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', output)); + .toFile(path.join(ThumbUtil.thumbPath, output)); } - static generateThumbnailForVideo(filename) { - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) + static generateThumbnailForVideo(filename, output) { + const filePath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename); + + ffmpeg(filePath) .thumbnail({ timestamps: [0], filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'), + folder: ThumbUtil.squareThumbPath, size: '64x64' }) .on('error', error => log.error(error.message)); - ffmpeg(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)) + + ffmpeg(filePath) .thumbnail({ timestamps: [0], filename: '%b.png', - folder: path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'), + folder: ThumbUtil.thumbPath, size: '150x?' }) .on('error', error => log.error(error.message)); + + try { + generatePreview({ + input: filePath, + width: 150, + output: path.join(ThumbUtil.videoPreviewPath, output) + }); + } catch (e) { + console.error(e); + } } static getFileThumbnail(filename) { if (!filename) return null; const ext = path.extname(filename).toLowerCase(); - if (!imageExtensions.includes(ext) && !videoExtensions.includes(ext)) return null; + if (!ThumbUtil.imageExtensions.includes(ext) && !ThumbUtil.videoExtensions.includes(ext)) return null; return `${filename.slice(0, -ext.length)}.png`; } + + static async removeThumbs(thumbName) { + await jetpack.removeAsync(path.join(ThumbUtil.thumbPath, thumbName)); + await jetpack.removeAsync(ThumbUtil.squareThumbPath, thumbName); + } } module.exports = ThumbUtil; diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 8c9428d..c997581 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -24,6 +24,8 @@ const ThumbUtil = require('./ThumbUtil'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); class Util { + static uploadPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER); + static uuid() { return uuidv4(); } @@ -55,7 +57,7 @@ class Util { }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem - const exists = jetpack.exists(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); + const exists = jetpack.exists(path.join(Util.uploadPath, filename)); if (!exists) return filename; if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for file'); @@ -86,10 +88,7 @@ class Util { } static async getFileHash(filename) { - const file = await jetpack.readAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename), - 'buffer' - ); + const file = await jetpack.readAsync(path.join(Util.uploadPath, filename), 'buffer'); if (!file) { log.error(`There was an error reading the file < ${filename} > for hashing`); return null; @@ -115,13 +114,9 @@ class Util { static async deleteFile(filename, deleteFromDB = false) { const thumbName = ThumbUtil.getFileThumbnail(filename); try { - await jetpack.removeAsync(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename)); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', thumbName) - ); - await jetpack.removeAsync( - path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square', thumbName) - ); + await jetpack.removeAsync(path.join(Util.uploadPath, filename)); + await ThumbUtil.removeThumbs(thumbName); + if (deleteFromDB) { await db .table('files') @@ -205,7 +200,7 @@ class Util { try { const zip = new Zip(); for (const file of files) { - zip.addLocalFile(path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, file)); + zip.addLocalFile(path.join(Util.uploadPath, file)); } zip.writeZip( path.join( -- cgit v1.2.3 From 24157dd1b944a2cfdc64f3ad0ae7cf9c2015145b Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 20:13:05 +0300 Subject: feat: experimental video preview generator in webm form --- src/api/utils/PreviewUtil.js | 74 ++++++++++++++++++++++++++++++++++++++++++++ src/api/utils/ThumbUtil.js | 11 ++++--- 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 src/api/utils/PreviewUtil.js (limited to 'src/api') diff --git a/src/api/utils/PreviewUtil.js b/src/api/utils/PreviewUtil.js new file mode 100644 index 0000000..bf3a480 --- /dev/null +++ b/src/api/utils/PreviewUtil.js @@ -0,0 +1,74 @@ +const ffmpeg = require('fluent-ffmpeg'); +const probe = require('ffmpeg-probe'); + +const path = require('path'); + +const noop = () => {}; + +module.exports = async opts => { + const { + log = noop, + + // general output options + quality = 2, + width, + height, + input, + output, + + numFrames, + numFramesPercent = 0.05 + } = opts; + + const info = await probe(input); + // const numFramesTotal = parseInt(info.streams[0].nb_frames, 10); + const { avg_frame_rate: avgFrameRate, duration } = info.streams[0]; + const [frames, time] = avgFrameRate.split('/').map(e => parseInt(e, 10)); + + const numFramesTotal = (frames / time) * duration; + + let numFramesToCapture = numFrames || numFramesPercent * numFramesTotal; + numFramesToCapture = Math.max(1, Math.min(numFramesTotal, numFramesToCapture)) | 0; + const nthFrame = (numFramesTotal / numFramesToCapture) | 0; + + const result = { + output, + numFrames: numFramesToCapture + }; + + await new Promise((resolve, reject) => { + let scale = null; + + if (width && height) { + result.width = width | 0; + result.height = height | 0; + scale = `scale=${width}:${height}`; + } else if (width) { + result.width = width | 0; + result.height = ((info.height * width) / info.width) | 0; + scale = `scale=${width}:-1`; + } else if (height) { + result.height = height | 0; + result.width = ((info.width * height) / info.height) | 0; + scale = `scale=-1:${height}`; + } else { + result.width = info.width; + result.height = info.height; + } + + const filter = [`select=not(mod(n\\,${nthFrame}))`, scale].filter(Boolean).join(','); + + ffmpeg(input) + .outputOptions(['-vsync', 'vfr']) + .outputOptions(['-q:v', quality, '-vf', filter]) + .outputOption('-an') + .outputFormat('webm') + .output(output) + .on('start', cmd => log && log({ cmd })) + .on('end', () => resolve()) + .on('error', err => reject(err)) + .run(); + }); + + return result; +}; diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index e508969..5cfd9c0 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -2,7 +2,7 @@ const jetpack = require('fs-jetpack'); const path = require('path'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); -const generatePreview = require('ffmpeg-generate-video-preview'); +const previewUtil = require('./PreviewUtil'); const log = require('./Log'); @@ -17,7 +17,7 @@ class ThumbUtil { static generateThumbnails(filename) { const ext = path.extname(filename).toLowerCase(); const output = `${filename.slice(0, -ext.length)}.png`; - const previewOutput = `${filename.slice(0, -ext.length)}.gif`; + const previewOutput = `${filename.slice(0, -ext.length)}.webm`; if (ThumbUtil.imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); if (ThumbUtil.videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename, previewOutput); @@ -38,7 +38,7 @@ class ThumbUtil { .toFile(path.join(ThumbUtil.thumbPath, output)); } - static generateThumbnailForVideo(filename, output) { + static async generateThumbnailForVideo(filename, output) { const filePath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, filename); ffmpeg(filePath) @@ -60,10 +60,11 @@ class ThumbUtil { .on('error', error => log.error(error.message)); try { - generatePreview({ + await previewUtil({ input: filePath, width: 150, - output: path.join(ThumbUtil.videoPreviewPath, output) + output: path.join(ThumbUtil.videoPreviewPath, output), + log: console.log }); } catch (e) { console.error(e); -- cgit v1.2.3 From dd46f79550d8e7a2f7a0364cc0fb8e7a38ed4aba Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 23:40:35 +0300 Subject: feat: return APIKey when fetching user --- src/api/routes/user/userGET.js | 3 ++- src/api/structures/Route.js | 32 ++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/user/userGET.js b/src/api/routes/user/userGET.js index fe46fd4..7929aac 100644 --- a/src/api/routes/user/userGET.js +++ b/src/api/routes/user/userGET.js @@ -11,7 +11,8 @@ class usersGET extends Route { user: { id: user.id, username: user.username, - isAdmin: user.isAdmin + isAdmin: user.isAdmin, + apiKey: user.apiKey } }); } diff --git a/src/api/structures/Route.js b/src/api/structures/Route.js index 8956c24..2402481 100644 --- a/src/api/structures/Route.js +++ b/src/api/structures/Route.js @@ -7,7 +7,7 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: nodePath.join(__dirname, '..', '..', '..', 'database.sqlite') + filename: nodePath.join(__dirname, '../../../database.sqlite') }, postProcessResponse: result => { /* @@ -16,11 +16,7 @@ const db = require('knex')({ some things like different data types for booleans need to be considered like in the implementation below where sqlite returns 1 and 0 instead of true and false. */ - const booleanFields = [ - 'enabled', - 'enableDownload', - 'isAdmin' - ]; + const booleanFields = ['enabled', 'enableDownload', 'isAdmin']; const processResponse = row => { Object.keys(row).forEach(key => { @@ -52,7 +48,10 @@ class Route { } async authorize(req, res) { - const banned = await db.table('bans').where({ ip: req.ip }).first(); + const banned = await db + .table('bans') + .where({ ip: req.ip }) + .first(); if (banned) return res.status(401).json({ message: 'This IP has been banned from using the service.' }); if (this.options.bypassAuth) return this.run(req, res, db); @@ -72,11 +71,16 @@ class Route { const id = decoded ? decoded.sub : ''; const iat = decoded ? decoded.iat : ''; - const user = await db.table('users').where({ id }).first(); + const user = await db + .table('users') + .where({ id }) + .first(); if (!user) return res.status(401).json({ message: 'Invalid authorization' }); - if (iat && iat < moment(user.passwordEditedAt).format('x')) return res.status(401).json({ message: 'Token expired' }); + if (iat && iat < moment(user.passwordEditedAt).format('x')) + return res.status(401).json({ message: 'Token expired' }); if (!user.enabled) return res.status(401).json({ message: 'This account has been disabled' }); - if (this.options.adminOnly && !user.isAdmin) return res.status(401).json({ message: 'Invalid authorization' }); + if (this.options.adminOnly && !user.isAdmin) + return res.status(401).json({ message: 'Invalid authorization' }); return this.run(req, res, db, user); }); @@ -84,14 +88,18 @@ class Route { async authorizeApiKey(req, res, apiKey) { if (!this.options.canApiKey) return res.status(401).json({ message: 'Api Key not allowed for this resource' }); - const user = await db.table('users').where({ apiKey }).first(); + const user = await db + .table('users') + .where({ apiKey }) + .first(); if (!user) return res.status(401).json({ message: 'Invalid authorization' }); if (!user.enabled) return res.status(401).json({ message: 'This account has been disabled' }); return this.run(req, res, db, user); } - run(req, res, db) { // eslint-disable-line no-unused-vars + run(req, res, db) { + // eslint-disable-line no-unused-vars return; } -- cgit v1.2.3 From 22f9eb4dff9ee03b5ec655db2204050ffe7a7771 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 2 Jul 2020 23:42:44 +0300 Subject: feat: refactor preview to support random fragment extraction --- src/api/utils/PreviewUtil.js | 74 ------------------ src/api/utils/ThumbUtil.js | 2 +- src/api/utils/videoPreview/FragmentPreview.js | 87 ++++++++++++++++++++++ src/api/utils/videoPreview/FrameIntervalPreview.js | 72 ++++++++++++++++++ 4 files changed, 160 insertions(+), 75 deletions(-) delete mode 100644 src/api/utils/PreviewUtil.js create mode 100644 src/api/utils/videoPreview/FragmentPreview.js create mode 100644 src/api/utils/videoPreview/FrameIntervalPreview.js (limited to 'src/api') diff --git a/src/api/utils/PreviewUtil.js b/src/api/utils/PreviewUtil.js deleted file mode 100644 index bf3a480..0000000 --- a/src/api/utils/PreviewUtil.js +++ /dev/null @@ -1,74 +0,0 @@ -const ffmpeg = require('fluent-ffmpeg'); -const probe = require('ffmpeg-probe'); - -const path = require('path'); - -const noop = () => {}; - -module.exports = async opts => { - const { - log = noop, - - // general output options - quality = 2, - width, - height, - input, - output, - - numFrames, - numFramesPercent = 0.05 - } = opts; - - const info = await probe(input); - // const numFramesTotal = parseInt(info.streams[0].nb_frames, 10); - const { avg_frame_rate: avgFrameRate, duration } = info.streams[0]; - const [frames, time] = avgFrameRate.split('/').map(e => parseInt(e, 10)); - - const numFramesTotal = (frames / time) * duration; - - let numFramesToCapture = numFrames || numFramesPercent * numFramesTotal; - numFramesToCapture = Math.max(1, Math.min(numFramesTotal, numFramesToCapture)) | 0; - const nthFrame = (numFramesTotal / numFramesToCapture) | 0; - - const result = { - output, - numFrames: numFramesToCapture - }; - - await new Promise((resolve, reject) => { - let scale = null; - - if (width && height) { - result.width = width | 0; - result.height = height | 0; - scale = `scale=${width}:${height}`; - } else if (width) { - result.width = width | 0; - result.height = ((info.height * width) / info.width) | 0; - scale = `scale=${width}:-1`; - } else if (height) { - result.height = height | 0; - result.width = ((info.width * height) / info.height) | 0; - scale = `scale=-1:${height}`; - } else { - result.width = info.width; - result.height = info.height; - } - - const filter = [`select=not(mod(n\\,${nthFrame}))`, scale].filter(Boolean).join(','); - - ffmpeg(input) - .outputOptions(['-vsync', 'vfr']) - .outputOptions(['-q:v', quality, '-vf', filter]) - .outputOption('-an') - .outputFormat('webm') - .output(output) - .on('start', cmd => log && log({ cmd })) - .on('end', () => resolve()) - .on('error', err => reject(err)) - .run(); - }); - - return result; -}; diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index 5cfd9c0..2931d3b 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -2,7 +2,7 @@ const jetpack = require('fs-jetpack'); const path = require('path'); const sharp = require('sharp'); const ffmpeg = require('fluent-ffmpeg'); -const previewUtil = require('./PreviewUtil'); +const previewUtil = require('./videoPreview/FragmentPreview'); const log = require('./Log'); diff --git a/src/api/utils/videoPreview/FragmentPreview.js b/src/api/utils/videoPreview/FragmentPreview.js new file mode 100644 index 0000000..8815392 --- /dev/null +++ b/src/api/utils/videoPreview/FragmentPreview.js @@ -0,0 +1,87 @@ +const ffmpeg = require('fluent-ffmpeg'); +const probe = require('ffmpeg-probe'); + +const noop = () => {}; + +const getRandomInt = (min, max) => { + const minInt = Math.ceil(min); + const maxInt = Math.floor(max); + + // eslint-disable-next-line no-mixed-operators + return Math.floor(Math.random() * (maxInt - minInt + 1) + minInt); +}; + +const getStartTime = (vDuration, fDuration, ignoreBeforePercent, ignoreAfterPercent) => { + // by subtracting the fragment duration we can be sure that the resulting + // start time + fragment duration will be less than the video duration + const safeVDuration = vDuration - fDuration; + + // if the fragment duration is longer than the video duration + if (safeVDuration <= 0) { + return 0; + } + + return getRandomInt(ignoreBeforePercent * safeVDuration, ignoreAfterPercent * safeVDuration); +}; + +module.exports = async opts => { + const { + log = noop, + + // general output options + quality = 2, + width, + height, + input, + output, + + fragmentDurationSecond = 3, + ignoreBeforePercent = 0.25, + ignoreAfterPercent = 0.75 + } = opts; + + const info = await probe(input); + + let { duration } = info.format; + duration = parseInt(duration, 10); + + const startTime = getStartTime(duration, fragmentDurationSecond, ignoreBeforePercent, ignoreAfterPercent); + + const result = { startTime, duration }; + + await new Promise((resolve, reject) => { + let scale = null; + + if (width && height) { + result.width = width | 0; + result.height = height | 0; + scale = `scale=${width}:${height}`; + } else if (width) { + result.width = width | 0; + result.height = ((info.height * width) / info.width) | 0; + scale = `scale=${width}:-1`; + } else if (height) { + result.height = height | 0; + result.width = ((info.width * height) / info.height) | 0; + scale = `scale=-1:${height}`; + } else { + result.width = info.width; + result.height = info.height; + } + + return ffmpeg() + .input(input) + .inputOptions([`-ss ${startTime}`]) + .outputOptions(['-vsync', 'vfr']) + .outputOptions(['-q:v', quality, '-vf', scale]) + .outputOptions([`-t ${fragmentDurationSecond}`]) + .noAudio() + .output(output) + .on('start', cmd => log && log({ cmd })) + .on('end', resolve) + .on('error', reject) + .run(); + }); + + return result; +}; diff --git a/src/api/utils/videoPreview/FrameIntervalPreview.js b/src/api/utils/videoPreview/FrameIntervalPreview.js new file mode 100644 index 0000000..75f6d2b --- /dev/null +++ b/src/api/utils/videoPreview/FrameIntervalPreview.js @@ -0,0 +1,72 @@ +const ffmpeg = require('fluent-ffmpeg'); +const probe = require('ffmpeg-probe'); + +const noop = () => {}; + +module.exports = async opts => { + const { + log = noop, + + // general output options + quality = 2, + width, + height, + input, + output, + + numFrames, + numFramesPercent = 0.05 + } = opts; + + const info = await probe(input); + // const numFramesTotal = parseInt(info.streams[0].nb_frames, 10); + const { avg_frame_rate: avgFrameRate, duration } = info.streams[0]; + const [frames, time] = avgFrameRate.split('/').map(e => parseInt(e, 10)); + + const numFramesTotal = (frames / time) * duration; + + let numFramesToCapture = numFrames || numFramesPercent * numFramesTotal; + numFramesToCapture = Math.max(1, Math.min(numFramesTotal, numFramesToCapture)) | 0; + const nthFrame = (numFramesTotal / numFramesToCapture) | 0; + + const result = { + output, + numFrames: numFramesToCapture + }; + + await new Promise((resolve, reject) => { + let scale = null; + + if (width && height) { + result.width = width | 0; + result.height = height | 0; + scale = `scale=${width}:${height}`; + } else if (width) { + result.width = width | 0; + result.height = ((info.height * width) / info.width) | 0; + scale = `scale=${width}:-1`; + } else if (height) { + result.height = height | 0; + result.width = ((info.width * height) / info.height) | 0; + scale = `scale=-1:${height}`; + } else { + result.width = info.width; + result.height = info.height; + } + + const filter = [`select=not(mod(n\\,${nthFrame}))`, scale].filter(Boolean).join(','); + + ffmpeg(input) + .outputOptions(['-vsync', 'vfr']) + .outputOptions(['-q:v', quality, '-vf', filter]) + .noAudio() + .outputFormat('webm') + .output(output) + .on('start', cmd => log && log({ cmd })) + .on('end', () => resolve()) + .on('error', err => reject(err)) + .run(); + }); + + return result; +}; -- cgit v1.2.3 From b620aa981546cb42e19f64d5349a9372e3d0c269 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sat, 4 Jul 2020 03:25:04 +0300 Subject: feat: refactor some of the queries to returned added/updated data --- src/api/routes/albums/albumFullGET.js | 13 +++++++++---- src/api/routes/albums/albumPOST.js | 17 ++++++++++++----- src/api/routes/albums/albumsGET.js | 25 +++++++++++++------------ src/api/routes/albums/link/linkEditPOST.js | 21 +++++++++++++-------- src/api/routes/albums/link/linkPOST.js | 24 +++++++++++++++++------- 5 files changed, 64 insertions(+), 36 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/albums/albumFullGET.js b/src/api/routes/albums/albumFullGET.js index cf434e4..a5c9707 100644 --- a/src/api/routes/albums/albumFullGET.js +++ b/src/api/routes/albums/albumFullGET.js @@ -10,22 +10,27 @@ class albumGET extends Route { const { id } = req.params; if (!id) return res.status(400).json({ message: 'Invalid id supplied' }); - const album = await db.table('albums').where({ id, userId: user.id }).first(); + const album = await db + .table('albums') + .where({ id, userId: user.id }) + .first(); if (!album) return res.status(404).json({ message: 'Album not found' }); let count = 0; - let files = db.table('albumsFiles') + let files = db + .table('albumsFiles') .where({ albumId: id }) .join('files', 'albumsFiles.fileId', 'files.id') - .select('files.id', 'files.name') + .select('files.id', 'files.name', 'files.createdAt') .orderBy('files.id', 'desc'); const { page, limit = 100 } = req.query; if (page && page >= 0) { files = await files.offset((page - 1) * limit).limit(limit); - const dbRes = await db.table('albumsFiles') + const dbRes = await db + .table('albumsFiles') .count('* as count') .where({ albumId: id }) .first(); diff --git a/src/api/routes/albums/albumPOST.js b/src/api/routes/albums/albumPOST.js index 0d3a44c..841da3d 100644 --- a/src/api/routes/albums/albumPOST.js +++ b/src/api/routes/albums/albumPOST.js @@ -14,18 +14,25 @@ class albumPOST extends Route { /* Check that an album with that name doesn't exist yet */ - const album = await db.table('albums').where({ name, userId: user.id }).first(); - if (album) return res.status(401).json({ message: 'There\'s already an album with that name' }); + const album = await db + .table('albums') + .where({ name, userId: user.id }) + .first(); + if (album) return res.status(401).json({ message: "There's already an album with that name" }); const now = moment.utc().toDate(); - await db.table('albums').insert({ + const insertObj = { name, userId: user.id, createdAt: now, editedAt: now - }); + }; - return res.json({ message: 'The album was created successfully' }); + const dbRes = await db.table('albums').insert(insertObj); + + insertObj.id = dbRes.pop(); + + return res.json({ message: 'The album was created successfully', data: insertObj }); } } diff --git a/src/api/routes/albums/albumsGET.js b/src/api/routes/albums/albumsGET.js index 1a7db87..569128c 100644 --- a/src/api/routes/albums/albumsGET.js +++ b/src/api/routes/albums/albumsGET.js @@ -12,30 +12,30 @@ class albumsGET extends Route { of the album files for displaying on the dashboard. It's probably useless for anyone consuming the API outside of the lolisafe frontend. */ - const albums = await db.table('albums') + const albums = await db + .table('albums') .where('albums.userId', user.id) - .select('id', 'name', 'editedAt'); + .select('id', 'name', 'createdAt', 'editedAt') + .orderBy('createdAt', 'desc'); for (const album of albums) { // TODO: Optimize the shit out of this. Ideally a JOIN that grabs all the needed stuff in 1 query instead of 3 // Fetch the total amount of files each album has. - const fileCount = await db.table('albumsFiles') // eslint-disable-line no-await-in-loop + const fileCount = await db + .table('albumsFiles') // eslint-disable-line no-await-in-loop .where('albumId', album.id) .count({ count: 'id' }); // Fetch the file list from each album but limit it to 5 per album - const filesToFetch = await db.table('albumsFiles') // eslint-disable-line no-await-in-loop + const files = await db + .table('albumsFiles') // eslint-disable-line no-await-in-loop + .join('files', { 'files.id': 'albumsFiles.fileId' }) .where('albumId', album.id) - .select('fileId') - .orderBy('id', 'desc') + .select('files.id', 'files.name') + .orderBy('albumsFiles.id', 'desc') .limit(5); - // Fetch the actual files - const files = await db.table('files') // eslint-disable-line no-await-in-loop - .whereIn('id', filesToFetch.map(el => el.fileId)) - .select('id', 'name'); - // Fetch thumbnails and stuff for (let file of files) { file = Util.constructFilePublicLink(file); @@ -58,7 +58,8 @@ class albumsDropdownGET extends Route { } async run(req, res, db, user) { - const albums = await db.table('albums') + const albums = await db + .table('albums') .where('userId', user.id) .select('id', 'name'); return res.json({ diff --git a/src/api/routes/albums/link/linkEditPOST.js b/src/api/routes/albums/link/linkEditPOST.js index 6776b73..0c7233b 100644 --- a/src/api/routes/albums/link/linkEditPOST.js +++ b/src/api/routes/albums/link/linkEditPOST.js @@ -14,17 +14,22 @@ class linkEditPOST extends Route { /* Make sure the link exists */ - const link = await db.table('links').where({ identifier, userId: user.id }).first(); - if (!link) return res.status(400).json({ message: 'The link doesn\'t exist or doesn\'t belong to the user' }); + const link = await db + .table('links') + .where({ identifier, userId: user.id }) + .first(); + if (!link) return res.status(400).json({ message: "The link doesn't exist or doesn't belong to the user" }); try { - await db.table('links') + const updateObj = { + enableDownload: enableDownload || false, + expiresAt // This one should be null if not supplied + }; + await db + .table('links') .where({ identifier }) - .update({ - enableDownload: enableDownload || false, - expiresAt // This one should be null if not supplied - }); - return res.json({ message: 'Editing the link was successful' }); + .update(updateObj); + return res.json({ message: 'Editing the link was successful', data: updateObj }); } catch (error) { return super.error(res, error); } diff --git a/src/api/routes/albums/link/linkPOST.js b/src/api/routes/albums/link/linkPOST.js index 6009922..7ecc5cb 100644 --- a/src/api/routes/albums/link/linkPOST.js +++ b/src/api/routes/albums/link/linkPOST.js @@ -14,23 +14,32 @@ class linkPOST extends Route { /* Make sure the album exists */ - const exists = await db.table('albums').where({ id: albumId, userId: user.id }).first(); + const exists = await db + .table('albums') + .where({ id: albumId, userId: user.id }) + .first(); if (!exists) return res.status(400).json({ message: 'Album doesn\t exist' }); /* Count the amount of links created for that album already and error out if max was reached */ - const count = await db.table('links').where('albumId', albumId).count({ count: 'id' }); - if (count[0].count >= parseInt(process.env.MAX_LINKS_PER_ALBUM, 10)) return res.status(400).json({ message: 'Maximum links per album reached' }); + const count = await db + .table('links') + .where('albumId', albumId) + .count({ count: 'id' }) + .first(); + if (count >= parseInt(process.env.MAX_LINKS_PER_ALBUM, 10)) + return res.status(400).json({ message: 'Maximum links per album reached' }); /* Try to allocate a new identifier on the db */ const identifier = await Util.getUniqueAlbumIdentifier(); - if (!identifier) return res.status(500).json({ message: 'There was a problem allocating a link for your album' }); + if (!identifier) + return res.status(500).json({ message: 'There was a problem allocating a link for your album' }); try { - await db.table('links').insert({ + const insertObj = { identifier, userId: user.id, albumId, @@ -38,11 +47,12 @@ class linkPOST extends Route { enableDownload: true, expiresAt: null, views: 0 - }); + }; + await db.table('links').insert(insertObj); return res.json({ message: 'The link was created successfully', - identifier + data: insertObj }); } catch (error) { return super.error(res, error); -- cgit v1.2.3 From 836a01327de6b2af5604bb77a34bc3f73b972178 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sat, 4 Jul 2020 03:25:21 +0300 Subject: chore: add nsfw flag to migration --- src/api/database/migrations/20190221225812_initialMigration.js | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/api') diff --git a/src/api/database/migrations/20190221225812_initialMigration.js b/src/api/database/migrations/20190221225812_initialMigration.js index a27a08a..4bcea8d 100644 --- a/src/api/database/migrations/20190221225812_initialMigration.js +++ b/src/api/database/migrations/20190221225812_initialMigration.js @@ -16,6 +16,7 @@ exports.up = async knex => { table.increments(); table.integer('userId'); table.string('name'); + table.boolean('nsfw'); table.timestamp('zippedAt'); table.timestamp('createdAt'); table.timestamp('editedAt'); @@ -28,6 +29,7 @@ exports.up = async knex => { table.string('original'); table.string('type'); table.integer('size'); + table.boolean('nsfw'); table.string('hash'); table.string('ip'); table.timestamp('createdAt'); -- cgit v1.2.3 From 1e1f3fbb27976a34f53a4e8d250da34dad4e6c20 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sat, 4 Jul 2020 23:18:51 +0300 Subject: feat: experimental videos in grid --- src/api/utils/ThumbUtil.js | 8 +++++++- src/api/utils/Util.js | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index 2931d3b..b449a6c 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -72,10 +72,16 @@ class ThumbUtil { } static getFileThumbnail(filename) { + // TODO: refactor so we don't do the same compare multiple times (poor cpu cycles) if (!filename) return null; const ext = path.extname(filename).toLowerCase(); if (!ThumbUtil.imageExtensions.includes(ext) && !ThumbUtil.videoExtensions.includes(ext)) return null; - return `${filename.slice(0, -ext.length)}.png`; + if (ThumbUtil.imageExtensions.includes(ext)) return { thumb: `${filename.slice(0, -ext.length)}.png` }; + if (ThumbUtil.videoExtensions.includes(ext)) + return { + thumb: `${filename.slice(0, -ext.length)}.png`, + preview: `${filename.slice(0, -ext.length)}.webm` + }; } static async removeThumbs(thumbName) { diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index c997581..496ea18 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -40,10 +40,11 @@ class Util { the site and the API under the same domain. Pls fix. */ file.url = `${process.env.DOMAIN}/${file.name}`; - const thumb = ThumbUtil.getFileThumbnail(file.name); + const { thumb, preview } = ThumbUtil.getFileThumbnail(file.name) || {}; if (thumb) { file.thumb = `${process.env.DOMAIN}/thumbs/${thumb}`; file.thumbSquare = `${process.env.DOMAIN}/thumbs/square/${thumb}`; + file.preview = preview && `${process.env.DOMAIN}/thumbs/preview/${preview}`; } return file; } -- cgit v1.2.3 From 04fdd63cee5327f49e5e11d5837a9031027c34ef Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Sun, 5 Jul 2020 04:17:09 +0300 Subject: feat: refactor single album page to use vuex --- src/api/routes/albums/albumGET.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/albums/albumGET.js b/src/api/routes/albums/albumGET.js index 1bf3630..f40750b 100644 --- a/src/api/routes/albums/albumGET.js +++ b/src/api/routes/albums/albumGET.js @@ -21,7 +21,7 @@ class albumGET extends Route { const files = await db.table('albumsFiles') .where({ albumId: link.albumId }) .join('files', 'albumsFiles.fileId', 'files.id') - .select('files.name') + .select('files.name', 'files.id') .orderBy('files.id', 'desc'); // Create the links for each file @@ -36,7 +36,7 @@ class albumGET extends Route { message: 'Successfully retrieved files', name: album.name, downloadEnabled: link.enableDownload, - files + files, }); } } -- cgit v1.2.3 From fb0bc57542a44dcc94149f393d8a4ff0c2e7902b Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Tue, 7 Jul 2020 02:02:59 +0300 Subject: feat: try fixing THE SHITTY WATERFALL --- src/api/routes/albums/albumFullGET.js | 3 ++- src/api/routes/albums/albumGET.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/api') diff --git a/src/api/routes/albums/albumFullGET.js b/src/api/routes/albums/albumFullGET.js index a5c9707..2c3a790 100644 --- a/src/api/routes/albums/albumFullGET.js +++ b/src/api/routes/albums/albumFullGET.js @@ -41,6 +41,7 @@ class albumGET extends Route { count = files.length; } + // eslint-disable-next-line no-restricted-syntax for (let file of files) { file = Util.constructFilePublicLink(file); } @@ -49,7 +50,7 @@ class albumGET extends Route { message: 'Successfully retrieved album', name: album.name, files, - count + count, }); } } diff --git a/src/api/routes/albums/albumGET.js b/src/api/routes/albums/albumGET.js index f40750b..81edc95 100644 --- a/src/api/routes/albums/albumGET.js +++ b/src/api/routes/albums/albumGET.js @@ -25,6 +25,7 @@ class albumGET extends Route { .orderBy('files.id', 'desc'); // Create the links for each file + // eslint-disable-next-line no-restricted-syntax for (let file of files) { file = Util.constructFilePublicLink(file); } -- cgit v1.2.3 From eccbb1ca93f1b86e9bc93dcbc1ec0ee9b168d949 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 02:32:12 +0300 Subject: fix: errors in Util caused by separating into different classes improperly --- src/api/generateThumbs.js | 1 + src/api/utils/Log.js | 5 +++++ src/api/utils/ThumbUtil.js | 41 +++++++++++++++++++++++++---------------- src/api/utils/Util.js | 24 ++++++++++++++---------- 4 files changed, 45 insertions(+), 26 deletions(-) (limited to 'src/api') diff --git a/src/api/generateThumbs.js b/src/api/generateThumbs.js index 1f2c531..0377fe7 100644 --- a/src/api/generateThumbs.js +++ b/src/api/generateThumbs.js @@ -9,6 +9,7 @@ const start = async () => { const files = fs.readdirSync(path.join(__dirname, '..', '..', process.env.UPLOAD_FOLDER)); for (const fileName of files) { console.log(`Generating thumb for '${fileName}`); + // eslint-disable-next-line no-await-in-loop await ThumbUtil.generateThumbnails(fileName); } }; diff --git a/src/api/utils/Log.js b/src/api/utils/Log.js index 6753f9e..8b38c2b 100644 --- a/src/api/utils/Log.js +++ b/src/api/utils/Log.js @@ -22,6 +22,11 @@ class Log { else console.log(chalk.red(args)); // eslint-disable-line no-console } + static debug(args) { + if (this.checkIfArrayOrObject(args)) dump(args); + else console.log(chalk.gray(args)); // eslint-disable-line no-console + } + /* static dump(args) { dump(args); diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index b449a6c..f8c73e7 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -8,19 +8,22 @@ const log = require('./Log'); class ThumbUtil { static imageExtensions = ['.jpg', '.jpeg', '.gif', '.png', '.webp']; + static videoExtensions = ['.webm', '.mp4', '.wmv', '.avi', '.mov']; - static thumbPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs'); - static squareThumbPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'square'); - static videoPreviewPath = path.join(__dirname, '..', '..', '..', process.env.UPLOAD_FOLDER, 'thumbs', 'preview'); + static thumbPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER, 'thumbs'); + + static squareThumbPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER, 'thumbs', 'square'); + + static videoPreviewPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER, 'thumbs', 'preview'); static generateThumbnails(filename) { const ext = path.extname(filename).toLowerCase(); const output = `${filename.slice(0, -ext.length)}.png`; const previewOutput = `${filename.slice(0, -ext.length)}.webm`; - if (ThumbUtil.imageExtensions.includes(ext)) return this.generateThumbnailForImage(filename, output); - if (ThumbUtil.videoExtensions.includes(ext)) return this.generateThumbnailForVideo(filename, previewOutput); + if (ThumbUtil.imageExtensions.includes(ext)) return ThumbUtil.generateThumbnailForImage(filename, output); + if (ThumbUtil.videoExtensions.includes(ext)) return ThumbUtil.generateThumbnailForVideo(filename, previewOutput); return null; } @@ -46,28 +49,28 @@ class ThumbUtil { timestamps: [0], filename: '%b.png', folder: ThumbUtil.squareThumbPath, - size: '64x64' + size: '64x64', }) - .on('error', error => log.error(error.message)); + .on('error', (error) => log.error(error.message)); ffmpeg(filePath) .thumbnail({ timestamps: [0], filename: '%b.png', folder: ThumbUtil.thumbPath, - size: '150x?' + size: '150x?', }) - .on('error', error => log.error(error.message)); + .on('error', (error) => log.error(error.message)); try { await previewUtil({ input: filePath, width: 150, output: path.join(ThumbUtil.videoPreviewPath, output), - log: console.log + log: log.debug, }); } catch (e) { - console.error(e); + log.error(e); } } @@ -77,16 +80,22 @@ class ThumbUtil { const ext = path.extname(filename).toLowerCase(); if (!ThumbUtil.imageExtensions.includes(ext) && !ThumbUtil.videoExtensions.includes(ext)) return null; if (ThumbUtil.imageExtensions.includes(ext)) return { thumb: `${filename.slice(0, -ext.length)}.png` }; - if (ThumbUtil.videoExtensions.includes(ext)) + if (ThumbUtil.videoExtensions.includes(ext)) { return { thumb: `${filename.slice(0, -ext.length)}.png`, - preview: `${filename.slice(0, -ext.length)}.webm` + preview: `${filename.slice(0, -ext.length)}.webm`, }; + } } - static async removeThumbs(thumbName) { - await jetpack.removeAsync(path.join(ThumbUtil.thumbPath, thumbName)); - await jetpack.removeAsync(ThumbUtil.squareThumbPath, thumbName); + static async removeThumbs({ thumb, preview }) { + if (thumb) { + await jetpack.removeAsync(path.join(ThumbUtil.thumbPath, thumb)); + await jetpack.removeAsync(path.join(ThumbUtil.squareThumbPath, thumb)); + } + if (preview) { + await jetpack.removeAsync(path.join(ThumbUtil.videoPreviewPath, preview)); + } } } diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 496ea18..ab59c95 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); @@ -9,9 +10,9 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: path.join(__dirname, '..', '..', '..', 'database.sqlite') + filename: path.join(__dirname, '../../../database.sqlite'), }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite' ? true : false + useNullAsDefault: process.env.DB_CLIENT === 'sqlite', }); const moment = require('moment'); const crypto = require('crypto'); @@ -51,11 +52,10 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = - randomstring.generate({ - length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), - capitalization: 'lowercase' - }) + path.extname(name).toLowerCase(); + const filename = randomstring.generate({ + length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), + capitalization: 'lowercase', + }) + path.extname(name).toLowerCase(); // TODO: Change this to look for the file in the db instead of in the filesystem const exists = jetpack.exists(path.join(Util.uploadPath, filename)); @@ -71,7 +71,7 @@ class Util { const retry = async (i = 0) => { const identifier = randomstring.generate({ length: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), - capitalization: 'lowercase' + capitalization: 'lowercase', }); const exists = await db .table('links') @@ -138,7 +138,9 @@ class Util { .table('files') .where({ id: fileAlbum.fileId }) .first(); + if (!file) continue; + await this.deleteFile(file.name, true); } } catch (error) { @@ -211,13 +213,15 @@ class Util { '..', process.env.UPLOAD_FOLDER, 'zips', - `${album.userId}-${album.id}.zip` - ) + `${album.userId}-${album.id}.zip`, + ), ); } catch (error) { log.error(error); } } + + static generateThumbnails = ThumbUtil.generateThumbnails; } module.exports = Util; -- cgit v1.2.3 From 1a8b6602e094289a4f477c33e432e0f5e1587b45 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 03:13:51 +0300 Subject: refactor: change uploader component to use vuex --- src/api/routes/albums/albumsGET.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/albums/albumsGET.js b/src/api/routes/albums/albumsGET.js index 569128c..c9ab025 100644 --- a/src/api/routes/albums/albumsGET.js +++ b/src/api/routes/albums/albumsGET.js @@ -1,3 +1,4 @@ +/* eslint-disable max-classes-per-file */ const Route = require('../../structures/Route'); const Util = require('../../utils/Util'); @@ -19,17 +20,15 @@ class albumsGET extends Route { .orderBy('createdAt', 'desc'); for (const album of albums) { - // TODO: Optimize the shit out of this. Ideally a JOIN that grabs all the needed stuff in 1 query instead of 3 - // Fetch the total amount of files each album has. - const fileCount = await db - .table('albumsFiles') // eslint-disable-line no-await-in-loop + const fileCount = await db // eslint-disable-line no-await-in-loop + .table('albumsFiles') .where('albumId', album.id) .count({ count: 'id' }); // Fetch the file list from each album but limit it to 5 per album - const files = await db - .table('albumsFiles') // eslint-disable-line no-await-in-loop + const files = await db // eslint-disable-line no-await-in-loop + .table('albumsFiles') .join('files', { 'files.id': 'albumsFiles.fileId' }) .where('albumId', album.id) .select('files.id', 'files.name') @@ -47,7 +46,7 @@ class albumsGET extends Route { return res.json({ message: 'Successfully retrieved albums', - albums + albums, }); } } @@ -64,7 +63,7 @@ class albumsDropdownGET extends Route { .select('id', 'name'); return res.json({ message: 'Successfully retrieved albums', - albums + albums, }); } } -- cgit v1.2.3 From 49d3e3b203ee287a53beb2a04faa8bf38ace6834 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 03:15:27 +0300 Subject: feat: add morgan for logging requests if env is not production --- src/api/structures/Route.js | 20 +++++++++----------- src/api/structures/Server.js | 40 ++++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 19 deletions(-) (limited to 'src/api') diff --git a/src/api/structures/Route.js b/src/api/structures/Route.js index 2402481..c2ad32e 100644 --- a/src/api/structures/Route.js +++ b/src/api/structures/Route.js @@ -7,9 +7,9 @@ const db = require('knex')({ user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_DATABASE, - filename: nodePath.join(__dirname, '../../../database.sqlite') + filename: nodePath.join(__dirname, '../../../database.sqlite'), }, - postProcessResponse: result => { + postProcessResponse: (result) => { /* Fun fact: Depending on the database used by the user and given that I don't want to force a specific database for everyone because of the nature of this project, @@ -18,8 +18,8 @@ const db = require('knex')({ */ const booleanFields = ['enabled', 'enableDownload', 'isAdmin']; - const processResponse = row => { - Object.keys(row).forEach(key => { + const processResponse = (row) => { + Object.keys(row).forEach((key) => { if (booleanFields.includes(key)) { if (row[key] === 0) row[key] = false; else if (row[key] === 1) row[key] = true; @@ -28,11 +28,11 @@ const db = require('knex')({ return row; }; - if (Array.isArray(result)) return result.map(row => processResponse(row)); + if (Array.isArray(result)) return result.map((row) => processResponse(row)); if (typeof result === 'object') return processResponse(result); return result; }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite3' ? true : false + useNullAsDefault: process.env.DB_CLIENT === 'sqlite3', }); const moment = require('moment'); const log = require('../utils/Log'); @@ -76,11 +76,9 @@ class Route { .where({ id }) .first(); if (!user) return res.status(401).json({ message: 'Invalid authorization' }); - if (iat && iat < moment(user.passwordEditedAt).format('x')) - return res.status(401).json({ message: 'Token expired' }); + if (iat && iat < moment(user.passwordEditedAt).format('x')) { return res.status(401).json({ message: 'Token expired' }); } if (!user.enabled) return res.status(401).json({ message: 'This account has been disabled' }); - if (this.options.adminOnly && !user.isAdmin) - return res.status(401).json({ message: 'Invalid authorization' }); + if (this.options.adminOnly && !user.isAdmin) { return res.status(401).json({ message: 'Invalid authorization' }); } return this.run(req, res, db, user); }); @@ -100,7 +98,7 @@ class Route { run(req, res, db) { // eslint-disable-line no-unused-vars - return; + } error(res, error) { diff --git a/src/api/structures/Server.js b/src/api/structures/Server.js index a8eccd9..5d2290b 100644 --- a/src/api/structures/Server.js +++ b/src/api/structures/Server.js @@ -1,6 +1,5 @@ require('dotenv').config(); -const log = require('../utils/Log'); const express = require('express'); const helmet = require('helmet'); const cors = require('cors'); @@ -8,11 +7,14 @@ const RateLimit = require('express-rate-limit'); const bodyParser = require('body-parser'); const jetpack = require('fs-jetpack'); const path = require('path'); +const morgan = require('morgan'); +const log = require('../utils/Log'); +const ThumbUtil = require('../utils/ThumbUtil'); const rateLimiter = new RateLimit({ windowMs: parseInt(process.env.RATE_LIMIT_WINDOW, 10), max: parseInt(process.env.RATE_LIMIT_MAX, 10), - delayMs: 0 + delayMs: 0, }); class Server { @@ -32,16 +34,38 @@ class Server { }); this.server.use(bodyParser.urlencoded({ extended: true })); this.server.use(bodyParser.json()); + if (process.env.NODE_ENV !== 'production') { + this.server.use(morgan('combined', { + skip(req) { + let ext = req.path.split('.').pop(); + if (ext) { ext = `.${ext.toLowerCase()}`; } + + if ( + ThumbUtil.imageExtensions.indexOf(ext) > -1 + || ThumbUtil.videoExtensions.indexOf(ext) > -1 + || req.path.indexOf('_nuxt') > -1 + || req.path.indexOf('favicon.ico') > -1 + ) { + return true; + } + return false; + }, + 'stream': { + write(str) { log.debug(str); }, + }, + })); + } // this.server.use(rateLimiter); // Serve the uploads - this.server.use(express.static(path.join(__dirname, '..', '..', '..', 'uploads'))); - this.routesFolder = path.join(__dirname, '..', 'routes'); + this.server.use(express.static(path.join(__dirname, '../../../uploads'))); + this.routesFolder = path.join(__dirname, '../routes'); } registerAllTheRoutes() { - jetpack.find(this.routesFolder, { matching: '*.js' }).forEach(routeFile => { - const RouteClass = require(path.join('..', '..', '..', routeFile)); + jetpack.find(this.routesFolder, { matching: '*.js' }).forEach((routeFile) => { + // eslint-disable-next-line import/no-dynamic-require, global-require + const RouteClass = require(path.join('../../../', routeFile)); let routes = [RouteClass]; if (Array.isArray(RouteClass)) routes = RouteClass; for (const File of routes) { @@ -55,7 +79,7 @@ class Server { serveNuxt() { // Serve the frontend if we are in production mode if (process.env.NODE_ENV === 'production') { - this.server.use(express.static(path.join(__dirname, '..', '..', '..', 'dist'))); + this.server.use(express.static(path.join(__dirname, '../../../dist'))); } /* @@ -66,7 +90,7 @@ class Server { */ this.server.all('*', (_req, res) => { try { - res.sendFile(path.join(__dirname, '..', '..', '..', 'dist', 'index.html')); + res.sendFile(path.join(__dirname, '../../../dist/index.html')); } catch (error) { res.json({ success: false, message: 'Something went wrong' }); } -- cgit v1.2.3 From b519b6ccb469e874c783b995ddf0ab6fabdb5a0e Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 03:37:50 +0300 Subject: refactor: refactor grid to use vuex for every action --- src/api/routes/files/albumAddPOST.js | 3 ++- src/api/routes/files/albumDelPOST.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/files/albumAddPOST.js b/src/api/routes/files/albumAddPOST.js index af39caa..a88e636 100644 --- a/src/api/routes/files/albumAddPOST.js +++ b/src/api/routes/files/albumAddPOST.js @@ -24,7 +24,8 @@ class albumAddPOST extends Route { } return res.json({ - message: 'Successfully added file to album' + message: 'Successfully added file to album', + data: { fileId, album: { id: album.id, name: album.name } }, }); } } diff --git a/src/api/routes/files/albumDelPOST.js b/src/api/routes/files/albumDelPOST.js index 9a4b87b..6e4d576 100644 --- a/src/api/routes/files/albumDelPOST.js +++ b/src/api/routes/files/albumDelPOST.js @@ -25,7 +25,8 @@ class albumDelPOST extends Route { } return res.json({ - message: 'Successfully removed file from album' + message: 'Successfully removed file from album', + data: { fileId, album: { id: album.id, name: album.name } }, }); } } -- cgit v1.2.3 From ad852de51a0d2dd5d29c08838d5a430c58849e74 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 04:00:12 +0300 Subject: chore: linter the entire project using the new rules --- .../migrations/20190221225812_initialMigration.js | 22 +++++------ src/api/database/seeds/initial.js | 5 ++- src/api/databaseMigration.js | 35 +++++++++-------- src/api/routes/admin/banIP.js | 2 +- src/api/routes/admin/fileGET.js | 2 +- src/api/routes/admin/unBanIP.js | 2 +- src/api/routes/admin/userDemote.js | 2 +- src/api/routes/admin/userDisable.js | 2 +- src/api/routes/admin/userEnable.js | 2 +- src/api/routes/admin/userGET.js | 2 +- src/api/routes/admin/userPromote.js | 2 +- src/api/routes/admin/userPurge.js | 2 +- src/api/routes/admin/usersGET.js | 2 +- src/api/routes/albums/albumDELETE.js | 1 - src/api/routes/albums/albumPOST.js | 4 +- src/api/routes/albums/albumZipGET.js | 10 ++--- src/api/routes/albums/link/linkDELETE.js | 3 +- src/api/routes/albums/link/linkEditPOST.js | 3 +- src/api/routes/albums/link/linkPOST.js | 10 ++--- src/api/routes/albums/link/linksGET.js | 2 +- src/api/routes/auth/loginPOST.js | 8 ++-- src/api/routes/auth/registerPOST.js | 8 ++-- src/api/routes/files/filesAlbumsGET.js | 4 +- src/api/routes/files/filesGET.js | 2 +- src/api/routes/files/tagAddPOST.js | 5 ++- src/api/routes/service/configGET.js | 10 ++--- src/api/routes/tags/tagPOST.js | 4 +- src/api/routes/tags/tagsGET.js | 3 +- src/api/routes/uploads/chunksPOST.js | 14 +++---- src/api/routes/uploads/uploadPOST.js | 44 +++++++++++----------- src/api/routes/user/apiKey.js | 6 +-- src/api/routes/user/changePasswordPOST.js | 6 +-- src/api/routes/user/userGET.js | 4 +- src/api/routes/verifyGET.js | 4 +- src/api/structures/Route.js | 8 ++-- src/api/structures/Server.js | 1 + src/api/utils/Log.js | 6 --- src/api/utils/ThumbUtil.js | 14 +++++-- src/api/utils/Util.js | 4 +- src/api/utils/videoPreview/FragmentPreview.js | 7 ++-- src/api/utils/videoPreview/FrameIntervalPreview.js | 13 ++++--- 41 files changed, 146 insertions(+), 144 deletions(-) (limited to 'src/api') diff --git a/src/api/database/migrations/20190221225812_initialMigration.js b/src/api/database/migrations/20190221225812_initialMigration.js index 4bcea8d..dd18ee5 100644 --- a/src/api/database/migrations/20190221225812_initialMigration.js +++ b/src/api/database/migrations/20190221225812_initialMigration.js @@ -1,5 +1,5 @@ -exports.up = async knex => { - await knex.schema.createTable('users', table => { +exports.up = async (knex) => { + await knex.schema.createTable('users', (table) => { table.increments(); table.string('username'); table.text('password'); @@ -12,7 +12,7 @@ exports.up = async knex => { table.timestamp('editedAt'); }); - await knex.schema.createTable('albums', table => { + await knex.schema.createTable('albums', (table) => { table.increments(); table.integer('userId'); table.string('name'); @@ -22,7 +22,7 @@ exports.up = async knex => { table.timestamp('editedAt'); }); - await knex.schema.createTable('files', table => { + await knex.schema.createTable('files', (table) => { table.increments(); table.integer('userId'); table.string('name'); @@ -36,7 +36,7 @@ exports.up = async knex => { table.timestamp('editedAt'); }); - await knex.schema.createTable('links', table => { + await knex.schema.createTable('links', (table) => { table.increments(); table.integer('userId'); table.integer('albumId'); @@ -49,19 +49,19 @@ exports.up = async knex => { table.timestamp('editedAt'); }); - await knex.schema.createTable('albumsFiles', table => { + await knex.schema.createTable('albumsFiles', (table) => { table.increments(); table.integer('albumId'); table.integer('fileId'); }); - await knex.schema.createTable('albumsLinks', table => { + await knex.schema.createTable('albumsLinks', (table) => { table.increments(); table.integer('albumId'); table.integer('linkId'); }); - await knex.schema.createTable('tags', table => { + await knex.schema.createTable('tags', (table) => { table.increments(); table.string('uuid'); table.integer('userId'); @@ -70,19 +70,19 @@ exports.up = async knex => { table.timestamp('editedAt'); }); - await knex.schema.createTable('fileTags', table => { + await knex.schema.createTable('fileTags', (table) => { table.increments(); table.integer('fileId'); table.integer('tagId'); }); - await knex.schema.createTable('bans', table => { + await knex.schema.createTable('bans', (table) => { table.increments(); table.string('ip'); table.timestamp('createdAt'); }); }; -exports.down = async knex => { +exports.down = async (knex) => { await knex.schema.dropTableIfExists('users'); await knex.schema.dropTableIfExists('albums'); await knex.schema.dropTableIfExists('files'); diff --git a/src/api/database/seeds/initial.js b/src/api/database/seeds/initial.js index 280fd74..cdbfa80 100644 --- a/src/api/database/seeds/initial.js +++ b/src/api/database/seeds/initial.js @@ -1,7 +1,8 @@ +/* eslint-disable no-console */ const bcrypt = require('bcrypt'); const moment = require('moment'); -exports.seed = async db => { +exports.seed = async (db) => { const now = moment.utc().toDate(); const user = await db.table('users').where({ username: process.env.ADMIN_ACCOUNT }).first(); if (user) return; @@ -14,7 +15,7 @@ exports.seed = async db => { createdAt: now, editedAt: now, enabled: true, - isAdmin: true + isAdmin: true, }); console.log(); console.log('========================================================='); diff --git a/src/api/databaseMigration.js b/src/api/databaseMigration.js index 5cf4b39..d95605d 100644 --- a/src/api/databaseMigration.js +++ b/src/api/databaseMigration.js @@ -1,28 +1,31 @@ +/* eslint-disable eqeqeq */ +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-console */ const nodePath = require('path'); const moment = require('moment'); const oldDb = require('knex')({ client: 'sqlite3', connection: { - filename: nodePath.join(__dirname, '..', '..', 'db') + filename: nodePath.join(__dirname, '..', '..', 'db'), }, - useNullAsDefault: true + useNullAsDefault: true, }); const newDb = require('knex')({ client: 'sqlite3', connection: { - filename: nodePath.join(__dirname, '..', '..', 'database.sqlite') + filename: nodePath.join(__dirname, '..', '..', 'database.sqlite'), }, - postProcessResponse: result => { + postProcessResponse: (result) => { const booleanFields = [ 'enabled', 'enableDownload', - 'isAdmin' + 'isAdmin', ]; - const processResponse = row => { - Object.keys(row).forEach(key => { + const processResponse = (row) => { + Object.keys(row).forEach((key) => { if (booleanFields.includes(key)) { if (row[key] === 0) row[key] = false; else if (row[key] === 1) row[key] = true; @@ -31,11 +34,11 @@ const newDb = require('knex')({ return row; }; - if (Array.isArray(result)) return result.map(row => processResponse(row)); + if (Array.isArray(result)) return result.map((row) => processResponse(row)); if (typeof result === 'object') return processResponse(result); return result; }, - useNullAsDefault: true + useNullAsDefault: true, }); const start = async () => { @@ -49,13 +52,13 @@ const start = async () => { id: user.id, username: user.username, password: user.password, - enabled: user.enabled == 1 ? true : false, + enabled: user.enabled == 1, isAdmin: false, apiKey: user.token, passwordEditedAt: now, apiKeyEditedAt: now, createdAt: now, - editedAt: now + editedAt: now, }; await newDb.table('users').insert(userToInsert); } @@ -71,7 +74,7 @@ const start = async () => { name: album.name, zippedAt: album.zipGeneratedAt ? moment.unix(album.zipGeneratedAt).toDate() : null, createdAt: moment.unix(album.timestamp).toDate(), - editedAt: moment.unix(album.editedAt).toDate() + editedAt: moment.unix(album.editedAt).toDate(), }; const linkToInsert = { userId: album.userid, @@ -81,13 +84,13 @@ const start = async () => { enabled: true, enableDownload: true, createdAt: now, - editedAt: now + editedAt: now, }; await newDb.table('albums').insert(albumToInsert); const insertedId = await newDb.table('links').insert(linkToInsert); await newDb.table('albumsLinks').insert({ albumId: album.id, - linkId: insertedId[0] + linkId: insertedId[0], }); } console.log('Finished migrating albums...'); @@ -106,12 +109,12 @@ const start = async () => { hash: file.hash, ip: file.ip, createdAt: moment.unix(file.timestamp).toDate(), - editedAt: moment.unix(file.timestamp).toDate() + editedAt: moment.unix(file.timestamp).toDate(), }; filesToInsert.push(fileToInsert); albumsFilesToInsert.push({ albumId: file.albumid, - fileId: file.id + fileId: file.id, }); } await newDb.batchInsert('files', filesToInsert, 20); diff --git a/src/api/routes/admin/banIP.js b/src/api/routes/admin/banIP.js index 692880d..4dfe03c 100644 --- a/src/api/routes/admin/banIP.js +++ b/src/api/routes/admin/banIP.js @@ -17,7 +17,7 @@ class banIP extends Route { } return res.json({ - message: 'Successfully banned the ip' + message: 'Successfully banned the ip', }); } } diff --git a/src/api/routes/admin/fileGET.js b/src/api/routes/admin/fileGET.js index 3bb8da4..0d1b147 100644 --- a/src/api/routes/admin/fileGET.js +++ b/src/api/routes/admin/fileGET.js @@ -21,7 +21,7 @@ class filesGET extends Route { return res.json({ message: 'Successfully retrieved file', file, - user + user, }); } } diff --git a/src/api/routes/admin/unBanIP.js b/src/api/routes/admin/unBanIP.js index 493834b..725468c 100644 --- a/src/api/routes/admin/unBanIP.js +++ b/src/api/routes/admin/unBanIP.js @@ -19,7 +19,7 @@ class unBanIP extends Route { } return res.json({ - message: 'Successfully unbanned the ip' + message: 'Successfully unbanned the ip', }); } } diff --git a/src/api/routes/admin/userDemote.js b/src/api/routes/admin/userDemote.js index b430a48..3f6623d 100644 --- a/src/api/routes/admin/userDemote.js +++ b/src/api/routes/admin/userDemote.js @@ -20,7 +20,7 @@ class userDemote extends Route { } return res.json({ - message: 'Successfully demoted user' + message: 'Successfully demoted user', }); } } diff --git a/src/api/routes/admin/userDisable.js b/src/api/routes/admin/userDisable.js index e39c811..029e4af 100644 --- a/src/api/routes/admin/userDisable.js +++ b/src/api/routes/admin/userDisable.js @@ -20,7 +20,7 @@ class userDisable extends Route { } return res.json({ - message: 'Successfully disabled user' + message: 'Successfully disabled user', }); } } diff --git a/src/api/routes/admin/userEnable.js b/src/api/routes/admin/userEnable.js index cff622f..aca7a0b 100644 --- a/src/api/routes/admin/userEnable.js +++ b/src/api/routes/admin/userEnable.js @@ -20,7 +20,7 @@ class userEnable extends Route { } return res.json({ - message: 'Successfully enabled user' + message: 'Successfully enabled user', }); } } diff --git a/src/api/routes/admin/userGET.js b/src/api/routes/admin/userGET.js index 14a6c92..30c79f4 100644 --- a/src/api/routes/admin/userGET.js +++ b/src/api/routes/admin/userGET.js @@ -23,7 +23,7 @@ class usersGET extends Route { return res.json({ message: 'Successfully retrieved user', user, - files + files, }); } catch (error) { return super.error(res, error); diff --git a/src/api/routes/admin/userPromote.js b/src/api/routes/admin/userPromote.js index 4a5ed88..3e14cb7 100644 --- a/src/api/routes/admin/userPromote.js +++ b/src/api/routes/admin/userPromote.js @@ -20,7 +20,7 @@ class userPromote extends Route { } return res.json({ - message: 'Successfully promoted user' + message: 'Successfully promoted user', }); } } diff --git a/src/api/routes/admin/userPurge.js b/src/api/routes/admin/userPurge.js index 90f6ec9..8f61ff9 100644 --- a/src/api/routes/admin/userPurge.js +++ b/src/api/routes/admin/userPurge.js @@ -18,7 +18,7 @@ class userDemote extends Route { } return res.json({ - message: 'Successfully deleted the user\'s files' + message: 'Successfully deleted the user\'s files', }); } } diff --git a/src/api/routes/admin/usersGET.js b/src/api/routes/admin/usersGET.js index 52a707f..4e9b954 100644 --- a/src/api/routes/admin/usersGET.js +++ b/src/api/routes/admin/usersGET.js @@ -12,7 +12,7 @@ class usersGET extends Route { return res.json({ message: 'Successfully retrieved users', - users + users, }); } catch (error) { return super.error(res, error); diff --git a/src/api/routes/albums/albumDELETE.js b/src/api/routes/albums/albumDELETE.js index 4e6640e..f9c22e6 100644 --- a/src/api/routes/albums/albumDELETE.js +++ b/src/api/routes/albums/albumDELETE.js @@ -1,5 +1,4 @@ const Route = require('../../structures/Route'); -const Util = require('../../utils/Util'); class albumDELETE extends Route { constructor() { diff --git a/src/api/routes/albums/albumPOST.js b/src/api/routes/albums/albumPOST.js index 841da3d..94ee8a7 100644 --- a/src/api/routes/albums/albumPOST.js +++ b/src/api/routes/albums/albumPOST.js @@ -1,5 +1,5 @@ -const Route = require('../../structures/Route'); const moment = require('moment'); +const Route = require('../../structures/Route'); class albumPOST extends Route { constructor() { @@ -25,7 +25,7 @@ class albumPOST extends Route { name, userId: user.id, createdAt: now, - editedAt: now + editedAt: now, }; const dbRes = await db.table('albums').insert(insertObj); diff --git a/src/api/routes/albums/albumZipGET.js b/src/api/routes/albums/albumZipGET.js index a6ef6fd..bd74ef3 100644 --- a/src/api/routes/albums/albumZipGET.js +++ b/src/api/routes/albums/albumZipGET.js @@ -1,8 +1,8 @@ +const path = require('path'); +const jetpack = require('fs-jetpack'); const Route = require('../../structures/Route'); const Util = require('../../utils/Util'); const log = require('../../utils/Log'); -const path = require('path'); -const jetpack = require('fs-jetpack'); class albumGET extends Route { constructor() { @@ -21,7 +21,7 @@ class albumGET extends Route { .where({ identifier, enabled: true, - enableDownload: true + enableDownload: true, }) .first(); if (!link) return res.status(400).json({ message: 'The supplied identifier could not be found' }); @@ -64,11 +64,11 @@ class albumGET extends Route { /* Get the actual files */ - const fileIds = fileList.map(el => el.fileId); + const fileIds = fileList.map((el) => el.fileId); const files = await db.table('files') .whereIn('id', fileIds) .select('name'); - const filesToZip = files.map(el => el.name); + const filesToZip = files.map((el) => el.name); try { Util.createZip(filesToZip, album); diff --git a/src/api/routes/albums/link/linkDELETE.js b/src/api/routes/albums/link/linkDELETE.js index b02d0b4..0381b50 100644 --- a/src/api/routes/albums/link/linkDELETE.js +++ b/src/api/routes/albums/link/linkDELETE.js @@ -1,5 +1,4 @@ const Route = require('../../../structures/Route'); -const { dump } = require('dumper.js'); class linkDELETE extends Route { constructor() { @@ -28,7 +27,7 @@ class linkDELETE extends Route { } return res.json({ - message: 'Successfully deleted link' + message: 'Successfully deleted link', }); } } diff --git a/src/api/routes/albums/link/linkEditPOST.js b/src/api/routes/albums/link/linkEditPOST.js index 0c7233b..4e0e0e1 100644 --- a/src/api/routes/albums/link/linkEditPOST.js +++ b/src/api/routes/albums/link/linkEditPOST.js @@ -1,5 +1,4 @@ const Route = require('../../../structures/Route'); -const log = require('../../../utils/Log'); class linkEditPOST extends Route { constructor() { @@ -23,7 +22,7 @@ class linkEditPOST extends Route { try { const updateObj = { enableDownload: enableDownload || false, - expiresAt // This one should be null if not supplied + expiresAt, // This one should be null if not supplied }; await db .table('links') diff --git a/src/api/routes/albums/link/linkPOST.js b/src/api/routes/albums/link/linkPOST.js index 7ecc5cb..d58598a 100644 --- a/src/api/routes/albums/link/linkPOST.js +++ b/src/api/routes/albums/link/linkPOST.js @@ -28,15 +28,13 @@ class linkPOST extends Route { .where('albumId', albumId) .count({ count: 'id' }) .first(); - if (count >= parseInt(process.env.MAX_LINKS_PER_ALBUM, 10)) - return res.status(400).json({ message: 'Maximum links per album reached' }); + if (count >= parseInt(process.env.MAX_LINKS_PER_ALBUM, 10)) return res.status(400).json({ message: 'Maximum links per album reached' }); /* Try to allocate a new identifier on the db */ const identifier = await Util.getUniqueAlbumIdentifier(); - if (!identifier) - return res.status(500).json({ message: 'There was a problem allocating a link for your album' }); + if (!identifier) return res.status(500).json({ message: 'There was a problem allocating a link for your album' }); try { const insertObj = { @@ -46,13 +44,13 @@ class linkPOST extends Route { enabled: true, enableDownload: true, expiresAt: null, - views: 0 + views: 0, }; await db.table('links').insert(insertObj); return res.json({ message: 'The link was created successfully', - data: insertObj + data: insertObj, }); } catch (error) { return super.error(res, error); diff --git a/src/api/routes/albums/link/linksGET.js b/src/api/routes/albums/link/linksGET.js index edab49a..4487c26 100644 --- a/src/api/routes/albums/link/linksGET.js +++ b/src/api/routes/albums/link/linksGET.js @@ -14,7 +14,7 @@ class linkPOST extends Route { return res.json({ message: 'Successfully retrieved links', - links + links, }); } } diff --git a/src/api/routes/auth/loginPOST.js b/src/api/routes/auth/loginPOST.js index 205737a..5c7730c 100644 --- a/src/api/routes/auth/loginPOST.js +++ b/src/api/routes/auth/loginPOST.js @@ -1,7 +1,7 @@ -const Route = require('../../structures/Route'); const bcrypt = require('bcrypt'); const moment = require('moment'); const JWT = require('jsonwebtoken'); +const Route = require('../../structures/Route'); class loginPOST extends Route { constructor() { @@ -36,7 +36,7 @@ class loginPOST extends Route { const jwt = JWT.sign({ iss: 'lolisafe', sub: user.id, - iat: moment.utc().valueOf() + iat: moment.utc().valueOf(), }, process.env.SECRET, { expiresIn: '30d' }); return res.json({ @@ -45,10 +45,10 @@ class loginPOST extends Route { id: user.id, username: user.username, apiKey: user.apiKey, - isAdmin: user.isAdmin + isAdmin: user.isAdmin, }, token: jwt, - apiKey: user.apiKey + apiKey: user.apiKey, }); } } diff --git a/src/api/routes/auth/registerPOST.js b/src/api/routes/auth/registerPOST.js index feeb360..e2ac018 100644 --- a/src/api/routes/auth/registerPOST.js +++ b/src/api/routes/auth/registerPOST.js @@ -1,7 +1,7 @@ -const Route = require('../../structures/Route'); -const log = require('../../utils/Log'); const bcrypt = require('bcrypt'); const moment = require('moment'); +const Route = require('../../structures/Route'); +const log = require('../../utils/Log'); class registerPOST extends Route { constructor() { @@ -9,7 +9,7 @@ class registerPOST extends Route { } async run(req, res, db) { - if (process.env.USER_ACCOUNTS == 'false') return res.status(401).json({ message: 'Creation of new accounts is currently disabled' }); + if (process.env.USER_ACCOUNTS === 'false') return res.status(401).json({ message: 'Creation of new accounts is currently disabled' }); if (!req.body) return res.status(400).json({ message: 'No body provided' }); const { username, password } = req.body; if (!username || !password) return res.status(401).json({ message: 'Invalid body provided' }); @@ -50,7 +50,7 @@ class registerPOST extends Route { createdAt: now, editedAt: now, enabled: true, - isAdmin: false + isAdmin: false, }); return res.json({ message: 'The account was created successfully' }); } diff --git a/src/api/routes/files/filesAlbumsGET.js b/src/api/routes/files/filesAlbumsGET.js index 7f1190c..f5f2f3b 100644 --- a/src/api/routes/files/filesAlbumsGET.js +++ b/src/api/routes/files/filesAlbumsGET.js @@ -18,7 +18,7 @@ class filesGET extends Route { .select('albumId'); if (albumFiles.length) { - albumFiles = albumFiles.map(a => a.albumId); + albumFiles = albumFiles.map((a) => a.albumId); albums = await db.table('albums') .whereIn('id', albumFiles) .select('id', 'name'); @@ -26,7 +26,7 @@ class filesGET extends Route { return res.json({ message: 'Successfully retrieved file albums', - albums + albums, }); } } diff --git a/src/api/routes/files/filesGET.js b/src/api/routes/files/filesGET.js index 9e90633..ce1d788 100644 --- a/src/api/routes/files/filesGET.js +++ b/src/api/routes/files/filesGET.js @@ -36,7 +36,7 @@ class filesGET extends Route { return res.json({ message: 'Successfully retrieved files', files, - count + count, }); } } diff --git a/src/api/routes/files/tagAddPOST.js b/src/api/routes/files/tagAddPOST.js index 25467ab..07ecb18 100644 --- a/src/api/routes/files/tagAddPOST.js +++ b/src/api/routes/files/tagAddPOST.js @@ -14,7 +14,8 @@ class tagAddPOST extends Route { const file = await db.table('files').where({ id: fileId, userId: user.id }).first(); if (!file) return res.status(400).json({ message: 'File doesn\'t exist.' }); - tagNames.forEach(async tag => { + // eslint-disable-next-line consistent-return + tagNames.forEach(async (tag) => { try { await db.table('fileTags').insert({ fileId, tag }); } catch (error) { @@ -23,7 +24,7 @@ class tagAddPOST extends Route { }); return res.json({ - message: 'Successfully added file to album' + message: 'Successfully added file to album', }); } } diff --git a/src/api/routes/service/configGET.js b/src/api/routes/service/configGET.js index b653066..3c6a2f8 100644 --- a/src/api/routes/service/configGET.js +++ b/src/api/routes/service/configGET.js @@ -15,11 +15,11 @@ class configGET extends Route { maxUploadSize: parseInt(process.env.MAX_SIZE, 10), filenameLength: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), albumLinkLength: parseInt(process.env.GENERATED_ALBUM_LENGTH, 10), - generateThumbnails: process.env.GENERATE_THUMBNAILS == 'true' ? true : false, - generateZips: process.env.GENERATE_ZIPS == 'true' ? true : false, - publicMode: process.env.PUBLIC_MODE == 'true' ? true : false, - enableAccounts: process.env.USER_ACCOUNTS == 'true' ? true : false - } + generateThumbnails: process.env.GENERATE_THUMBNAILS === 'true', + generateZips: process.env.GENERATE_ZIPS === 'true', + publicMode: process.env.PUBLIC_MODE === 'true', + enableAccounts: process.env.USER_ACCOUNTS === 'true', + }, }); } } diff --git a/src/api/routes/tags/tagPOST.js b/src/api/routes/tags/tagPOST.js index b6ec395..856e0d4 100644 --- a/src/api/routes/tags/tagPOST.js +++ b/src/api/routes/tags/tagPOST.js @@ -1,5 +1,5 @@ -const Route = require('../../structures/Route'); const moment = require('moment'); +const Route = require('../../structures/Route'); class tagPOST extends Route { constructor() { @@ -22,7 +22,7 @@ class tagPOST extends Route { name, userId: user.id, createdAt: now, - editedAt: now + editedAt: now, }); return res.json({ message: 'The tag was created successfully' }); diff --git a/src/api/routes/tags/tagsGET.js b/src/api/routes/tags/tagsGET.js index 871148e..848e08d 100644 --- a/src/api/routes/tags/tagsGET.js +++ b/src/api/routes/tags/tagsGET.js @@ -1,5 +1,4 @@ const Route = require('../../structures/Route'); -const Util = require('../../utils/Util'); class tagsGET extends Route { constructor() { @@ -20,7 +19,7 @@ class tagsGET extends Route { return res.json({ message: 'Successfully retrieved tags', - tags + tags, }); } catch (error) { return super.error(res, error); diff --git a/src/api/routes/uploads/chunksPOST.js b/src/api/routes/uploads/chunksPOST.js index 013c0d6..a9baf55 100644 --- a/src/api/routes/uploads/chunksPOST.js +++ b/src/api/routes/uploads/chunksPOST.js @@ -1,27 +1,27 @@ -const Route = require('../../structures/Route'); const path = require('path'); -const Util = require('../../utils/Util'); const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); +const Util = require('../../utils/Util'); +const Route = require('../../structures/Route'); class uploadPOST extends Route { constructor() { super('/upload/chunks', 'post', { bypassAuth: true, - canApiKey: true + canApiKey: true, }); } - async run(req, res, db) { + async run(req, res) { const filename = Util.getUniqueFilename(randomstring.generate(32)); // console.log('Files', req.body.files); const info = { size: req.body.files[0].size, - url: `${process.env.DOMAIN}/` + url: `${process.env.DOMAIN}/`, }; for (const chunk of req.body.files) { - const { uuid, count } = chunk; + const { uuid } = chunk; // console.log('Chunk', chunk); const chunkOutput = path.join(__dirname, @@ -65,7 +65,7 @@ class uploadPOST extends Route { return res.status(201).send({ message: 'Sucessfully merged the chunk(s).', - ...info + ...info, /* name: `${filename}${ext || ''}`, size: exists.size, diff --git a/src/api/routes/uploads/uploadPOST.js b/src/api/routes/uploads/uploadPOST.js index 6c01dd3..48fc592 100644 --- a/src/api/routes/uploads/uploadPOST.js +++ b/src/api/routes/uploads/uploadPOST.js @@ -1,17 +1,18 @@ -const Route = require('../../structures/Route'); const path = require('path'); -const Util = require('../../utils/Util'); const jetpack = require('fs-jetpack'); const multer = require('multer'); const moment = require('moment'); +const Util = require('../../utils/Util'); +const Route = require('../../structures/Route'); + const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: parseInt(process.env.MAX_SIZE, 10) * (1000 * 1000), - files: 1 + files: 1, }, - fileFilter: (req, file, cb) => { - // TODO: Enable blacklisting of files/extensions + fileFilter: (req, file, cb) => + // TODO: Enable blacklisting of files/extensions /* if (options.blacklist.mimes.includes(file.mimetype)) { return cb(new Error(`${file.mimetype} is a blacklisted filetype.`)); @@ -19,35 +20,34 @@ const upload = multer({ return cb(new Error(`${path.extname(file.originalname).toLowerCase()} is a blacklisted extension.`)); } */ - return cb(null, true); - } + cb(null, true) + , }).array('files[]'); /* TODO: If source has transparency generate a png thumbnail, otherwise a jpg. TODO: If source is a gif, generate a thumb of the first frame and play the gif on hover on the frontend. - TODO: If source is a video, generate a thumb of the first frame and save the video length to the file? - Another possible solution would be to play a gif on hover that grabs a few chunks like youtube. TODO: Think if its worth making a folder with the user uuid in uploads/ and upload the pictures there so that this way at least not every single file will be in 1 directory - - Addendum to this: Now that the default behaviour is to serve files with node, we can actually pull this off. Before this, having files in - subfolders meant messing with nginx and the paths, but now it should be fairly easy to re-arrange the folder structure with express.static - I see great value in this, open to suggestions. + XXX: Now that the default behaviour is to serve files with node, we can actually pull this off. + Before this, having files in subfolders meant messing with nginx and the paths, + but now it should be fairly easy to re-arrange the folder structure with express.static + I see great value in this, open to suggestions. */ class uploadPOST extends Route { constructor() { super('/upload', 'post', { bypassAuth: true, - canApiKey: true + canApiKey: true, }); } async run(req, res, db) { const user = await Util.isAuthorized(req); - if (!user && process.env.PUBLIC_MODE == 'false') return res.status(401).json({ message: 'Not authorized to use this resource' }); + if (!user && process.env.PUBLIC_MODE === 'false') return res.status(401).json({ message: 'Not authorized to use this resource' }); const albumId = req.body.albumid || req.headers.albumid; if (albumId && !user) return res.status(401).json({ message: 'Only registered users can upload files to an album' }); @@ -56,12 +56,13 @@ class uploadPOST extends Route { if (!album) return res.status(401).json({ message: 'Album doesn\'t exist or it doesn\'t belong to the user' }); } - return upload(req, res, async err => { + return upload(req, res, async (err) => { if (err) console.error(err.message); let uploadedFile = {}; let insertedId; + // eslint-disable-next-line no-underscore-dangle const remappedKeys = this._remapKeys(req.body); const file = req.files[0]; @@ -105,7 +106,7 @@ class uploadPOST extends Route { name: filename, hash, size: file.buffer.length, - url: filename + url: filename, }; } @@ -124,7 +125,7 @@ class uploadPOST extends Route { return res.status(201).send({ message: 'Sucessfully uploaded the file.', - ...uploadedFile + ...uploadedFile, }); }); } @@ -137,7 +138,7 @@ class uploadPOST extends Route { size: exists.size, url: `${process.env.DOMAIN}/${exists.name}`, deleteUrl: `${process.env.DOMAIN}/api/file/${exists.id}`, - repeated: true + repeated: true, }); return Util.deleteFile(filename); @@ -145,7 +146,7 @@ class uploadPOST extends Route { async checkIfFileExists(db, user, hash) { const exists = await db.table('files') - .where(function() { // eslint-disable-line func-names + .where(function () { // eslint-disable-line func-names if (user) this.where('userId', user.id); else this.whereNull('userId'); }) @@ -186,7 +187,7 @@ class uploadPOST extends Route { hash: file.hash, ip: req.ip, createdAt: now, - editedAt: now + editedAt: now, }); } else { insertedId = await db.table('files').insert({ @@ -198,7 +199,7 @@ class uploadPOST extends Route { hash: file.hash, ip: req.ip, createdAt: now, - editedAt: now + editedAt: now, }, 'id'); } return insertedId; @@ -220,6 +221,7 @@ class uploadPOST extends Route { } return body; } + return keys; } } diff --git a/src/api/routes/user/apiKey.js b/src/api/routes/user/apiKey.js index a87d98d..a63f0c0 100644 --- a/src/api/routes/user/apiKey.js +++ b/src/api/routes/user/apiKey.js @@ -1,7 +1,7 @@ -const Route = require('../../structures/Route'); const randomstring = require('randomstring'); const moment = require('moment'); const { dump } = require('dumper.js'); +const Route = require('../../structures/Route'); class apiKeyPOST extends Route { constructor() { @@ -17,7 +17,7 @@ class apiKeyPOST extends Route { .where({ id: user.id }) .update({ apiKey, - apiKeyEditedAt: now + apiKeyEditedAt: now, }); } catch (error) { dump(error); @@ -26,7 +26,7 @@ class apiKeyPOST extends Route { return res.json({ message: 'Successfully created new api key', - apiKey + apiKey, }); } } diff --git a/src/api/routes/user/changePasswordPOST.js b/src/api/routes/user/changePasswordPOST.js index 9cd621e..1b3a27a 100644 --- a/src/api/routes/user/changePasswordPOST.js +++ b/src/api/routes/user/changePasswordPOST.js @@ -1,7 +1,7 @@ -const Route = require('../../structures/Route'); -const log = require('../../utils/Log'); const bcrypt = require('bcrypt'); const moment = require('moment'); +const Route = require('../../structures/Route'); +const log = require('../../utils/Log'); class changePasswordPOST extends Route { constructor() { @@ -36,7 +36,7 @@ class changePasswordPOST extends Route { const now = moment.utc().toDate(); await db.table('users').where('id', user.id).update({ password: hash, - passwordEditedAt: now + passwordEditedAt: now, }); return res.json({ message: 'The password was changed successfully' }); diff --git a/src/api/routes/user/userGET.js b/src/api/routes/user/userGET.js index 7929aac..6f179a9 100644 --- a/src/api/routes/user/userGET.js +++ b/src/api/routes/user/userGET.js @@ -12,8 +12,8 @@ class usersGET extends Route { id: user.id, username: user.username, isAdmin: user.isAdmin, - apiKey: user.apiKey - } + apiKey: user.apiKey, + }, }); } } diff --git a/src/api/routes/verifyGET.js b/src/api/routes/verifyGET.js index 2f370e8..107c20a 100644 --- a/src/api/routes/verifyGET.js +++ b/src/api/routes/verifyGET.js @@ -11,8 +11,8 @@ class verifyGET extends Route { user: { id: user.id, username: user.username, - isAdmin: user.isAdmin - } + isAdmin: user.isAdmin, + }, }); } } diff --git a/src/api/structures/Route.js b/src/api/structures/Route.js index c2ad32e..400ae3d 100644 --- a/src/api/structures/Route.js +++ b/src/api/structures/Route.js @@ -55,7 +55,8 @@ class Route { if (banned) return res.status(401).json({ message: 'This IP has been banned from using the service.' }); if (this.options.bypassAuth) return this.run(req, res, db); - // The only reason I call it token here and not Api Key is to be backwards compatible with the uploader and sharex + // The only reason I call it token here and not Api Key is to be backwards compatible + // with the uploader and sharex // Small price to pay. if (req.headers.token) return this.authorizeApiKey(req, res, req.headers.token); if (!req.headers.authorization) return res.status(401).json({ message: 'No authorization header provided' }); @@ -96,10 +97,7 @@ class Route { return this.run(req, res, db, user); } - run(req, res, db) { - // eslint-disable-line no-unused-vars - - } + run() {} error(res, error) { log.error(error); diff --git a/src/api/structures/Server.js b/src/api/structures/Server.js index 5d2290b..c8537fb 100644 --- a/src/api/structures/Server.js +++ b/src/api/structures/Server.js @@ -11,6 +11,7 @@ const morgan = require('morgan'); const log = require('../utils/Log'); const ThumbUtil = require('../utils/ThumbUtil'); +// eslint-disable-next-line no-unused-vars const rateLimiter = new RateLimit({ windowMs: parseInt(process.env.RATE_LIMIT_WINDOW, 10), max: parseInt(process.env.RATE_LIMIT_MAX, 10), diff --git a/src/api/utils/Log.js b/src/api/utils/Log.js index 8b38c2b..99d11e4 100644 --- a/src/api/utils/Log.js +++ b/src/api/utils/Log.js @@ -27,12 +27,6 @@ class Log { else console.log(chalk.gray(args)); // eslint-disable-line no-console } - /* - static dump(args) { - dump(args); - } - */ - static checkIfArrayOrObject(thing) { if (typeof thing === typeof [] || typeof thing === typeof {}) return true; return false; diff --git a/src/api/utils/ThumbUtil.js b/src/api/utils/ThumbUtil.js index f8c73e7..98ba5c0 100644 --- a/src/api/utils/ThumbUtil.js +++ b/src/api/utils/ThumbUtil.js @@ -22,7 +22,9 @@ class ThumbUtil { const output = `${filename.slice(0, -ext.length)}.png`; const previewOutput = `${filename.slice(0, -ext.length)}.webm`; + // eslint-disable-next-line max-len if (ThumbUtil.imageExtensions.includes(ext)) return ThumbUtil.generateThumbnailForImage(filename, output); + // eslint-disable-next-line max-len if (ThumbUtil.videoExtensions.includes(ext)) return ThumbUtil.generateThumbnailForVideo(filename, previewOutput); return null; } @@ -75,17 +77,21 @@ class ThumbUtil { } static getFileThumbnail(filename) { - // TODO: refactor so we don't do the same compare multiple times (poor cpu cycles) if (!filename) return null; const ext = path.extname(filename).toLowerCase(); - if (!ThumbUtil.imageExtensions.includes(ext) && !ThumbUtil.videoExtensions.includes(ext)) return null; - if (ThumbUtil.imageExtensions.includes(ext)) return { thumb: `${filename.slice(0, -ext.length)}.png` }; - if (ThumbUtil.videoExtensions.includes(ext)) { + + const isImage = ThumbUtil.imageExtensions.includes(ext); + const isVideo = ThumbUtil.videoExtensions.includes(ext); + + if (isImage) return { thumb: `${filename.slice(0, -ext.length)}.png` }; + if (isVideo) { return { thumb: `${filename.slice(0, -ext.length)}.png`, preview: `${filename.slice(0, -ext.length)}.webm`, }; } + + return null; } static async removeThumbs({ thumb, preview }) { diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index ab59c95..905948a 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -52,7 +52,7 @@ class Util { static getUniqueFilename(name) { const retry = (i = 0) => { - const filename = randomstring.generate({ + const filename = randomstring.generate({ length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10), capitalization: 'lowercase', }) + path.extname(name).toLowerCase(); @@ -81,7 +81,7 @@ class Util { /* It's funny but if you do i++ the asignment never gets done resulting in an infinite loop */ - if (i < 5) return retry(++i); + if (i < 5) return retry(i + 1); log.error('Couldnt allocate identifier for album'); return null; }; diff --git a/src/api/utils/videoPreview/FragmentPreview.js b/src/api/utils/videoPreview/FragmentPreview.js index 8815392..bf623c1 100644 --- a/src/api/utils/videoPreview/FragmentPreview.js +++ b/src/api/utils/videoPreview/FragmentPreview.js @@ -1,3 +1,4 @@ +/* eslint-disable no-bitwise */ const ffmpeg = require('fluent-ffmpeg'); const probe = require('ffmpeg-probe'); @@ -24,7 +25,7 @@ const getStartTime = (vDuration, fDuration, ignoreBeforePercent, ignoreAfterPerc return getRandomInt(ignoreBeforePercent * safeVDuration, ignoreAfterPercent * safeVDuration); }; -module.exports = async opts => { +module.exports = async (opts) => { const { log = noop, @@ -37,7 +38,7 @@ module.exports = async opts => { fragmentDurationSecond = 3, ignoreBeforePercent = 0.25, - ignoreAfterPercent = 0.75 + ignoreAfterPercent = 0.75, } = opts; const info = await probe(input); @@ -77,7 +78,7 @@ module.exports = async opts => { .outputOptions([`-t ${fragmentDurationSecond}`]) .noAudio() .output(output) - .on('start', cmd => log && log({ cmd })) + .on('start', (cmd) => log && log({ cmd })) .on('end', resolve) .on('error', reject) .run(); diff --git a/src/api/utils/videoPreview/FrameIntervalPreview.js b/src/api/utils/videoPreview/FrameIntervalPreview.js index 75f6d2b..8c5f1c3 100644 --- a/src/api/utils/videoPreview/FrameIntervalPreview.js +++ b/src/api/utils/videoPreview/FrameIntervalPreview.js @@ -1,9 +1,10 @@ +/* eslint-disable no-bitwise */ const ffmpeg = require('fluent-ffmpeg'); const probe = require('ffmpeg-probe'); const noop = () => {}; -module.exports = async opts => { +module.exports = async (opts) => { const { log = noop, @@ -15,13 +16,13 @@ module.exports = async opts => { output, numFrames, - numFramesPercent = 0.05 + numFramesPercent = 0.05, } = opts; const info = await probe(input); // const numFramesTotal = parseInt(info.streams[0].nb_frames, 10); const { avg_frame_rate: avgFrameRate, duration } = info.streams[0]; - const [frames, time] = avgFrameRate.split('/').map(e => parseInt(e, 10)); + const [frames, time] = avgFrameRate.split('/').map((e) => parseInt(e, 10)); const numFramesTotal = (frames / time) * duration; @@ -31,7 +32,7 @@ module.exports = async opts => { const result = { output, - numFrames: numFramesToCapture + numFrames: numFramesToCapture, }; await new Promise((resolve, reject) => { @@ -62,9 +63,9 @@ module.exports = async opts => { .noAudio() .outputFormat('webm') .output(output) - .on('start', cmd => log && log({ cmd })) + .on('start', (cmd) => log && log({ cmd })) .on('end', () => resolve()) - .on('error', err => reject(err)) + .on('error', (err) => reject(err)) .run(); }); -- cgit v1.2.3 From 6713eca9d4a4887dc8d7416dbdd8ec37de7bb2ed Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Wed, 8 Jul 2020 19:22:25 +0300 Subject: chore: add unique integrity checks to the database for many-to-many tables --- .../migrations/20190221225812_initialMigration.js | 20 +++++++++++++++----- src/api/routes/uploads/uploadPOST.js | 10 ++-------- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'src/api') diff --git a/src/api/database/migrations/20190221225812_initialMigration.js b/src/api/database/migrations/20190221225812_initialMigration.js index dd18ee5..b755a33 100644 --- a/src/api/database/migrations/20190221225812_initialMigration.js +++ b/src/api/database/migrations/20190221225812_initialMigration.js @@ -1,11 +1,11 @@ exports.up = async (knex) => { await knex.schema.createTable('users', (table) => { table.increments(); - table.string('username'); + table.string('username').unique(); table.text('password'); table.boolean('enabled'); table.boolean('isAdmin'); - table.string('apiKey'); + table.string('apiKey').unique(); table.timestamp('passwordEditedAt'); table.timestamp('apiKeyEditedAt'); table.timestamp('createdAt'); @@ -16,10 +16,12 @@ exports.up = async (knex) => { table.increments(); table.integer('userId'); table.string('name'); - table.boolean('nsfw'); + table.boolean('nsfw').defaultTo(false); table.timestamp('zippedAt'); table.timestamp('createdAt'); table.timestamp('editedAt'); + + table.unique(['userId', 'name']); }); await knex.schema.createTable('files', (table) => { @@ -29,7 +31,7 @@ exports.up = async (knex) => { table.string('original'); table.string('type'); table.integer('size'); - table.boolean('nsfw'); + table.boolean('nsfw').defaultTo(false); table.string('hash'); table.string('ip'); table.timestamp('createdAt'); @@ -47,18 +49,22 @@ exports.up = async (knex) => { table.timestamp('expiresAt'); table.timestamp('createdAt'); table.timestamp('editedAt'); + + table.unique(['userId', 'albumId', 'identifier']); }); await knex.schema.createTable('albumsFiles', (table) => { table.increments(); table.integer('albumId'); table.integer('fileId'); + + table.unique(['albumId', 'fileId']); }); await knex.schema.createTable('albumsLinks', (table) => { table.increments(); table.integer('albumId'); - table.integer('linkId'); + table.integer('linkId').unique(); }); await knex.schema.createTable('tags', (table) => { @@ -68,12 +74,16 @@ exports.up = async (knex) => { table.string('name'); table.timestamp('createdAt'); table.timestamp('editedAt'); + + table.unique(['userId', 'name']); }); await knex.schema.createTable('fileTags', (table) => { table.increments(); table.integer('fileId'); table.integer('tagId'); + + table.unique(['fileId', 'tagId']); }); await knex.schema.createTable('bans', (table) => { diff --git a/src/api/routes/uploads/uploadPOST.js b/src/api/routes/uploads/uploadPOST.js index 48fc592..99f5ee5 100644 --- a/src/api/routes/uploads/uploadPOST.js +++ b/src/api/routes/uploads/uploadPOST.js @@ -84,10 +84,7 @@ class uploadPOST extends Route { if (remappedKeys && remappedKeys.uuid) { const chunkOutput = path.join(__dirname, - '..', - '..', - '..', - '..', + '../../../../', process.env.UPLOAD_FOLDER, 'chunks', remappedKeys.uuid, @@ -95,10 +92,7 @@ class uploadPOST extends Route { await jetpack.writeAsync(chunkOutput, file.buffer); } else { const output = path.join(__dirname, - '..', - '..', - '..', - '..', + '../../../../', process.env.UPLOAD_FOLDER, filename); await jetpack.writeAsync(output, file.buffer); -- cgit v1.2.3 From 746a4546122be2ed79ad5858de6ce2c686f78ef0 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 9 Jul 2020 02:22:08 +0300 Subject: fix: stop leaking user's password and their apikey to admins --- src/api/routes/admin/userGET.js | 5 ++++- src/api/structures/Route.js | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/admin/userGET.js b/src/api/routes/admin/userGET.js index 30c79f4..2fb80d1 100644 --- a/src/api/routes/admin/userGET.js +++ b/src/api/routes/admin/userGET.js @@ -11,7 +11,10 @@ class usersGET extends Route { if (!id) return res.status(400).json({ message: 'Invalid user ID supplied' }); try { - const user = await db.table('users').where({ id }).first(); + const user = await db.table('users') + .select('id, username, enabled, createdAt, editeadAt, apiKeyEditedAt, isAdmin') + .where({ id }) + .first(); const files = await db.table('files') .where({ userId: user.id }) .orderBy('id', 'desc'); diff --git a/src/api/structures/Route.js b/src/api/structures/Route.js index 400ae3d..6be0dc7 100644 --- a/src/api/structures/Route.js +++ b/src/api/structures/Route.js @@ -77,7 +77,9 @@ class Route { .where({ id }) .first(); if (!user) return res.status(401).json({ message: 'Invalid authorization' }); - if (iat && iat < moment(user.passwordEditedAt).format('x')) { return res.status(401).json({ message: 'Token expired' }); } + if (iat && iat < moment(user.passwordEditedAt).format('x')) { + return res.status(401).json({ message: 'Token expired' }); + } if (!user.enabled) return res.status(401).json({ message: 'This account has been disabled' }); if (this.options.adminOnly && !user.isAdmin) { return res.status(401).json({ message: 'Invalid authorization' }); } -- cgit v1.2.3 From 7e78a03931173437cd4aec5454663ee3cc3aee23 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Fri, 10 Jul 2020 01:13:23 +0300 Subject: fix: stop leaking user passwords to admins AGAIN --- src/api/routes/admin/fileGET.js | 5 ++++- src/api/routes/admin/userGET.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/routes/admin/fileGET.js b/src/api/routes/admin/fileGET.js index 0d1b147..239b128 100644 --- a/src/api/routes/admin/fileGET.js +++ b/src/api/routes/admin/fileGET.js @@ -11,7 +11,10 @@ class filesGET extends Route { if (!id) return res.status(400).json({ message: 'Invalid file ID supplied' }); let file = await db.table('files').where({ id }).first(); - const user = await db.table('users').where({ id: file.userId }).first(); + const user = await db.table('users') + .select('id', 'username', 'enabled', 'createdAt', 'editedAt', 'apiKeyEditedAt', 'isAdmin') + .where({ id: file.userId }) + .first(); file = Util.constructFilePublicLink(file); // Additional relevant data diff --git a/src/api/routes/admin/userGET.js b/src/api/routes/admin/userGET.js index 2fb80d1..f5f2508 100644 --- a/src/api/routes/admin/userGET.js +++ b/src/api/routes/admin/userGET.js @@ -12,7 +12,7 @@ class usersGET extends Route { try { const user = await db.table('users') - .select('id, username, enabled, createdAt, editeadAt, apiKeyEditedAt, isAdmin') + .select('id', 'username', 'enabled', 'createdAt', 'editedAt', 'apiKeyEditedAt', 'isAdmin') .where({ id }) .first(); const files = await db.table('files') -- cgit v1.2.3