From 53f5015c99b3040e955632525bde4ad70250af9a Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Tue, 5 Jan 2021 00:25:53 +0200 Subject: feat: check for real mimetype using file-type For now, if file-type returns undefined, we take the value from the browser. In the future this should be removed to ensure people can't bypass the real mime checking using a special file that can't be recognized by file-type. --- src/api/utils/Util.js | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index ae13eb5..9d5021d 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -35,6 +35,10 @@ class Util { return blockedExtensions.includes(extension); } + static getMimeFromType(fileTypeMimeObj) { + return fileTypeMimeObj ? fileTypeMimeObj.mime : undefined; + } + static constructFilePublicLink(file) { /* TODO: This wont work without a reverse proxy serving both @@ -225,6 +229,7 @@ class Util { static async storeFileToDb(req, res, user, file, db) { const dbFile = await db.table('files') + // eslint-disable-next-line func-names .where(function() { if (user === undefined) { this.whereNull('userId'); -- cgit v1.2.3 From f151a8ac3a8f944829e55fc007823b167f1a2597 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Tue, 5 Jan 2021 22:58:41 +0200 Subject: chore: Move statistics related functions to it's own file fix: Extract database constructor to a separate file to ensure we only create one knex instance/db feat: Add cron for saving the stats to the database every hour feat: Get cached stats from database (if a stat is not found in the db, generate it) --- src/api/utils/Util.js | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 9d5021d..8eafcff 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -3,27 +3,20 @@ const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); const JWT = require('jsonwebtoken'); -const db = require('knex')({ - client: process.env.DB_CLIENT, - connection: { - host: process.env.DB_HOST, - user: process.env.DB_USER, - password: process.env.DB_PASSWORD, - database: process.env.DB_DATABASE, - filename: path.join(__dirname, '../../../database/database.sqlite') - }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite' -}); +const db = require('../structures/Database'); const moment = require('moment'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); const log = require('./Log'); const ThumbUtil = require('./ThumbUtil'); +const StatsGenerator = require('./StatsGenerator'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz']; +let statsLastSavedTime = null; + class Util { static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER); @@ -316,6 +309,35 @@ class Util { return extname + multi; } + + static async saveStatsToDb() { + // if we alredy saved a stats to the db, and there were no new changes to the db since then + // skip generating and saving new stats. + // XXX: Should we save non-db related statistics to the database anyway? (like performance, disk usage) + if (statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime) { + return; + } + + const now = moment.utc().toDate(); + const stats = await StatsGenerator.getStats(db); + + let batchId = 1; + + const res = (await db('statistics').max({ lastBatch: 'batchId' }))[0]; + if (res && res.lastBatch) { + batchId = res.lastBatch + 1; + } + + try { + for (const [type, data] of Object.entries(stats)) { + await db.table('statistics').insert({ type, data: JSON.stringify(data), createdAt: now, batchId }); + } + + statsLastSavedTime = now.getTime(); + } catch (error) { + console.error(error); + } + } } module.exports = Util; -- cgit v1.2.3 From b3df1dd7a653909fb3f18ed7c0c91ffa347480fb Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 7 Jan 2021 21:36:56 +0200 Subject: feat: change mutation timestamp on every database mutation --- src/api/utils/Util.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 8eafcff..658ac61 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -99,7 +99,8 @@ class Util { await db .table('files') .where('name', filename) - .delete(); + .delete() + .wasMutated(); } } catch (error) { log.error(`There was an error removing the file < ${filename} >`); @@ -257,9 +258,9 @@ class Util { let fileId; if (process.env.DB_CLIENT === 'sqlite3') { - fileId = await db.table('files').insert(data); + fileId = await db.table('files').insert(data).wasMutated(); } else { - fileId = await db.table('files').insert(data, 'id'); + fileId = await db.table('files').insert(data, 'id').wasMutated(); } return { @@ -273,7 +274,7 @@ class Util { const now = moment.utc().toDate(); try { - await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }); + await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }).wasMutated(); await db.table('albums').where('id', albumId).update('editedAt', now); } catch (error) { console.error(error); -- cgit v1.2.3 From 925080f6a08a1f1515143db1bd6aef8109f5fb67 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Thu, 7 Jan 2021 23:55:37 +0200 Subject: chore: refactor stats generator to use an enum for types fix: saved data not being converted to JSON automatically when using SQLite (and possibly MSSQL) feat: display when a stat was generated in the section's header fix: not being able to click through the footer (sorry IE11 users, you still won't be able to click through it) fix: add is-mobile to columns so they don't stack on top of each other feat: add text next to the NSFW toggle and make it look like the elements around it (begone round iOS edges) --- src/api/utils/Util.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/api/utils/Util.js') diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 658ac61..6feedd4 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -311,11 +311,18 @@ class Util { return extname + multi; } - static async saveStatsToDb() { + // TODO: Allow choosing what to save to db and what stats we care about in general + // TODO: if a stat is not saved to db but selected to be shows on the dashboard, it will be generated during the request + static async saveStatsToDb(force) { + // If there were no changes since the instance started, don't generate new stats + // OR // if we alredy saved a stats to the db, and there were no new changes to the db since then // skip generating and saving new stats. - // XXX: Should we save non-db related statistics to the database anyway? (like performance, disk usage) - if (statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime) { + if (!force && + (!db.userParams.lastMutationTime || + (statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime) + ) + ) { return; } -- cgit v1.2.3