From f151a8ac3a8f944829e55fc007823b167f1a2597 Mon Sep 17 00:00:00 2001 From: Zephyrrus Date: Tue, 5 Jan 2021 22:58:41 +0200 Subject: chore: Move statistics related functions to it's own file fix: Extract database constructor to a separate file to ensure we only create one knex instance/db feat: Add cron for saving the stats to the database every hour feat: Get cached stats from database (if a stat is not found in the db, generate it) --- src/api/utils/StatsGenerator.js | 209 ++++++++++++++++++++++++++++++++++++++++ src/api/utils/Util.js | 44 ++++++--- 2 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 src/api/utils/StatsGenerator.js (limited to 'src/api/utils') diff --git a/src/api/utils/StatsGenerator.js b/src/api/utils/StatsGenerator.js new file mode 100644 index 0000000..2e48f32 --- /dev/null +++ b/src/api/utils/StatsGenerator.js @@ -0,0 +1,209 @@ +const si = require('systeminformation'); + +class StatsGenerator { + static statGenerators = { + system: StatsGenerator.getSystemInfo, + fileSystems: StatsGenerator.getFileSystemsInfo, + uploads: StatsGenerator.getUploadsInfo, + users: StatsGenerator.getUsersInfo, + albums: StatsGenerator.getAlbumStats + }; + + static keyOrder = Object.keys(StatsGenerator.statGenerators); + + static async getSystemInfo() { + const os = await si.osInfo(); + + const currentLoad = await si.currentLoad(); + const mem = await si.mem(); + const time = si.time(); + const nodeUptime = process.uptime(); + + return { + 'Platform': `${os.platform} ${os.arch}`, + 'Distro': `${os.distro} ${os.release}`, + 'Kernel': os.kernel, + 'CPU Load': `${currentLoad.currentload.toFixed(1)}%`, + 'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '), + 'System Memory': { + value: { + used: mem.active, + total: mem.total + }, + type: 'byteUsage' + }, + 'Memory Usage': { + value: process.memoryUsage().rss, + type: 'byte' + }, + 'System Uptime': { + value: time.uptime, + type: 'time' + }, + 'Node.js': `${process.versions.node}`, + 'Service Uptime': { + value: Math.floor(nodeUptime), + type: 'time' + } + }; + } + + static async getFileSystemsInfo() { + const stats = {}; + + const fsSize = await si.fsSize(); + for (const fs of fsSize) { + stats[`${fs.fs} (${fs.type}) on ${fs.mount}`] = { + value: { + total: fs.size, + used: fs.used + }, + type: 'byteUsage' + }; + } + + return stats; + } + + static async getUploadsInfo(db) { + const stats = { + 'Total': 0, + 'Images': 0, + 'Videos': 0, + 'Others': { + data: {}, + count: 0, + type: 'detailed' + }, + 'Size in DB': { + value: 0, + type: 'byte' + } + }; + + const getFilesCountAndSize = async () => { + const uploads = await db.table('files').select('size'); + + return { + 'Total': uploads.length, + 'Size in DB': { + value: uploads.reduce((acc, upload) => acc + parseInt(upload.size, 10), 0), + type: 'byte' + } + }; + }; + + const getImagesCount = async () => { + const Images = await db.table('files') + .where('type', 'like', `image/%`) + .count('id as count') + .then(rows => rows[0].count); + + return { Images }; + }; + + const getVideosCount = async () => { + const Videos = await db.table('files') + .where('type', 'like', `video/%`) + .count('id as count') + .then(rows => rows[0].count); + + return { Videos }; + }; + + const getOthersCount = async () => { + // rename to key, value from type, count + const data = await db.table('files') + .select('type as key') + .count('id as value') + .whereNot('type', 'like', `image/%`) + .whereNot('type', 'like', `video/%`) + .groupBy('key') + .orderBy('value', 'desc'); + + const count = data.reduce((acc, val) => acc + val.value, 0); + + return { + Others: { + data, + count, + type: 'detailed' + } + }; + }; + + const result = await Promise.all([getFilesCountAndSize(), getImagesCount(), getVideosCount(), getOthersCount()]); + + return { ...stats, ...Object.assign({}, ...result) }; + } + + static async getUsersInfo(db) { + const stats = { + Total: 0, + Admins: 0, + Disabled: 0 + }; + + const users = await db.table('users'); + stats.Total = users.length; + + for (const user of users) { + if (!user.enabled) { + stats.Disabled++; + } + + if (user.isAdmin) { + stats.Admins++; + } + } + + return stats; + } + + static async getAlbumStats(db) { + const stats = { + 'Total': 0, + 'NSFW': 0, + 'Generated archives': 0, + 'Generated identifiers': 0, + 'Files in albums': 0 + }; + + const albums = await db.table('albums'); + stats.Total = albums.length; + for (const album of albums) { + if (album.nsfw) stats.NSFW++; + if (album.zipGeneratedAt) stats['Generated archives']++; // XXX: Bobby checks each after if a zip really exists on the disk. Is it really needed? + } + + stats['Generated identifiers'] = await db.table('albumsLinks').count('id as count').then(rows => rows[0].count); + stats['Files in albums'] = await db.table('albumsFiles') + .whereNotNull('albumId') + .count('id as count') + .then(rows => rows[0].count); + + return stats; + } + + static async getStats(db) { + const res = {}; + + for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) { + res[name] = await funct(db); + } + + return res; + } + + static async getMissingStats(db, existingStats) { + const res = {}; + + for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) { + if (existingStats.indexOf(name) === -1) res[name] = await funct(db); + } + + return res; + } +} + +module.exports = StatsGenerator; diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 9d5021d..8eafcff 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -3,27 +3,20 @@ const jetpack = require('fs-jetpack'); const randomstring = require('randomstring'); const path = require('path'); const JWT = require('jsonwebtoken'); -const db = require('knex')({ - client: process.env.DB_CLIENT, - connection: { - host: process.env.DB_HOST, - user: process.env.DB_USER, - password: process.env.DB_PASSWORD, - database: process.env.DB_DATABASE, - filename: path.join(__dirname, '../../../database/database.sqlite') - }, - useNullAsDefault: process.env.DB_CLIENT === 'sqlite' -}); +const db = require('../structures/Database'); const moment = require('moment'); const Zip = require('adm-zip'); const uuidv4 = require('uuid/v4'); const log = require('./Log'); const ThumbUtil = require('./ThumbUtil'); +const StatsGenerator = require('./StatsGenerator'); const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(','); const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz']; +let statsLastSavedTime = null; + class Util { static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER); @@ -316,6 +309,35 @@ class Util { return extname + multi; } + + static async saveStatsToDb() { + // if we alredy saved a stats to the db, and there were no new changes to the db since then + // skip generating and saving new stats. + // XXX: Should we save non-db related statistics to the database anyway? (like performance, disk usage) + if (statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime) { + return; + } + + const now = moment.utc().toDate(); + const stats = await StatsGenerator.getStats(db); + + let batchId = 1; + + const res = (await db('statistics').max({ lastBatch: 'batchId' }))[0]; + if (res && res.lastBatch) { + batchId = res.lastBatch + 1; + } + + try { + for (const [type, data] of Object.entries(stats)) { + await db.table('statistics').insert({ type, data: JSON.stringify(data), createdAt: now, batchId }); + } + + statsLastSavedTime = now.getTime(); + } catch (error) { + console.error(error); + } + } } module.exports = Util; -- cgit v1.2.3