aboutsummaryrefslogtreecommitdiff
path: root/src/api/utils
diff options
context:
space:
mode:
Diffstat (limited to 'src/api/utils')
-rw-r--r--src/api/utils/StatsGenerator.js225
-rw-r--r--src/api/utils/Util.js65
-rw-r--r--src/api/utils/multerStorage.js56
3 files changed, 312 insertions, 34 deletions
diff --git a/src/api/utils/StatsGenerator.js b/src/api/utils/StatsGenerator.js
new file mode 100644
index 0000000..ce73cd2
--- /dev/null
+++ b/src/api/utils/StatsGenerator.js
@@ -0,0 +1,225 @@
+const si = require('systeminformation');
+
+class StatsGenerator {
+ // symbols would be better because they're unique, but harder to serialize them
+ static Type = Object.freeze({
+ // should contain key value: number
+ TIME: 'time',
+ // should contain key value: number
+ BYTE: 'byte',
+ // should contain key value: { used: number, total: number }
+ BYTE_USAGE: 'byteUsage',
+ // should contain key data: Array<{ key: string, value: number | string }>
+ // and optionally a count/total
+ DETAILED: 'detailed',
+ // hidden type should be skipped during iteration, can contain anything
+ // these should be treated on a case by case basis on the frontend
+ HIDDEN: 'hidden'
+ });
+
+ static statGenerators = {
+ system: StatsGenerator.getSystemInfo,
+ fileSystems: StatsGenerator.getFileSystemsInfo,
+ uploads: StatsGenerator.getUploadsInfo,
+ users: StatsGenerator.getUsersInfo,
+ albums: StatsGenerator.getAlbumStats
+ };
+
+ static keyOrder = Object.keys(StatsGenerator.statGenerators);
+
+ static async getSystemInfo() {
+ const os = await si.osInfo();
+
+ const currentLoad = await si.currentLoad();
+ const mem = await si.mem();
+ const time = si.time();
+ const nodeUptime = process.uptime();
+
+ return {
+ 'Platform': `${os.platform} ${os.arch}`,
+ 'Distro': `${os.distro} ${os.release}`,
+ 'Kernel': os.kernel,
+ 'CPU Load': `${currentLoad.currentload.toFixed(1)}%`,
+ 'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
+ 'System Memory': {
+ value: {
+ used: mem.active,
+ total: mem.total
+ },
+ type: StatsGenerator.Type.BYTE_USAGE
+ },
+ 'Memory Usage': {
+ value: process.memoryUsage().rss,
+ type: StatsGenerator.Type.BYTE
+ },
+ 'System Uptime': {
+ value: time.uptime,
+ type: StatsGenerator.Type.TIME
+ },
+ 'Node.js': `${process.versions.node}`,
+ 'Service Uptime': {
+ value: Math.floor(nodeUptime),
+ type: StatsGenerator.Type.TIME
+ }
+ };
+ }
+
+ static async getFileSystemsInfo() {
+ const stats = {};
+
+ const fsSize = await si.fsSize();
+ for (const fs of fsSize) {
+ stats[`${fs.fs} (${fs.type}) on ${fs.mount}`] = {
+ value: {
+ total: fs.size,
+ used: fs.used
+ },
+ type: StatsGenerator.Type.BYTE_USAGE
+ };
+ }
+
+ return stats;
+ }
+
+ static async getUploadsInfo(db) {
+ const stats = {
+ 'Total': 0,
+ 'Images': 0,
+ 'Videos': 0,
+ 'Others': {
+ data: {},
+ count: 0,
+ type: StatsGenerator.Type.DETAILED
+ },
+ 'Size in DB': {
+ value: 0,
+ type: StatsGenerator.Type.BYTE
+ }
+ };
+
+ const getFilesCountAndSize = async () => {
+ const uploads = await db.table('files').select('size');
+
+ return {
+ 'Total': uploads.length,
+ 'Size in DB': {
+ value: uploads.reduce((acc, upload) => acc + parseInt(upload.size, 10), 0),
+ type: StatsGenerator.Type.BYTE
+ }
+ };
+ };
+
+ const getImagesCount = async () => {
+ const Images = await db.table('files')
+ .where('type', 'like', `image/%`)
+ .count('id as count')
+ .then(rows => rows[0].count);
+
+ return { Images };
+ };
+
+ const getVideosCount = async () => {
+ const Videos = await db.table('files')
+ .where('type', 'like', `video/%`)
+ .count('id as count')
+ .then(rows => rows[0].count);
+
+ return { Videos };
+ };
+
+ const getOthersCount = async () => {
+ // rename to key, value from type, count
+ const data = await db.table('files')
+ .select('type as key')
+ .count('id as value')
+ .whereNot('type', 'like', `image/%`)
+ .whereNot('type', 'like', `video/%`)
+ .groupBy('key')
+ .orderBy('value', 'desc');
+
+ const count = data.reduce((acc, val) => acc + val.value, 0);
+
+ return {
+ Others: {
+ data,
+ count,
+ type: StatsGenerator.Type.DETAILED
+ }
+ };
+ };
+
+ const result = await Promise.all([getFilesCountAndSize(), getImagesCount(), getVideosCount(), getOthersCount()]);
+
+ return { ...stats, ...Object.assign({}, ...result) };
+ }
+
+ static async getUsersInfo(db) {
+ const stats = {
+ Total: 0,
+ Admins: 0,
+ Disabled: 0
+ };
+
+ const users = await db.table('users');
+ stats.Total = users.length;
+
+ for (const user of users) {
+ if (!user.enabled) {
+ stats.Disabled++;
+ }
+
+ if (user.isAdmin) {
+ stats.Admins++;
+ }
+ }
+
+ return stats;
+ }
+
+ static async getAlbumStats(db) {
+ const stats = {
+ 'Total': 0,
+ 'NSFW': 0,
+ 'Generated archives': 0,
+ 'Generated identifiers': 0,
+ 'Files in albums': 0
+ };
+
+ const albums = await db.table('albums');
+ stats.Total = albums.length;
+ for (const album of albums) {
+ if (album.nsfw) stats.NSFW++;
+ if (album.zipGeneratedAt) stats['Generated archives']++; // XXX: Bobby checks each after if a zip really exists on the disk. Is it really needed?
+ }
+
+ stats['Generated identifiers'] = await db.table('albumsLinks').count('id as count').then(rows => rows[0].count);
+ stats['Files in albums'] = await db.table('albumsFiles')
+ .whereNotNull('albumId')
+ .count('id as count')
+ .then(rows => rows[0].count);
+
+ return stats;
+ }
+
+ static async getStats(db) {
+ const res = {};
+
+ for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) {
+ res[name] = await funct(db);
+ }
+
+ return res;
+ }
+
+ static async getMissingStats(db, existingStats) {
+ const res = {};
+
+ for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) {
+ if (existingStats.indexOf(name) === -1) res[name] = await funct(db);
+ }
+
+ return res;
+ }
+}
+
+module.exports = StatsGenerator;
diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js
index ae13eb5..6feedd4 100644
--- a/src/api/utils/Util.js
+++ b/src/api/utils/Util.js
@@ -3,27 +3,20 @@ const jetpack = require('fs-jetpack');
const randomstring = require('randomstring');
const path = require('path');
const JWT = require('jsonwebtoken');
-const db = require('knex')({
- client: process.env.DB_CLIENT,
- connection: {
- host: process.env.DB_HOST,
- user: process.env.DB_USER,
- password: process.env.DB_PASSWORD,
- database: process.env.DB_DATABASE,
- filename: path.join(__dirname, '../../../database/database.sqlite')
- },
- useNullAsDefault: process.env.DB_CLIENT === 'sqlite'
-});
+const db = require('../structures/Database');
const moment = require('moment');
const Zip = require('adm-zip');
const uuidv4 = require('uuid/v4');
const log = require('./Log');
const ThumbUtil = require('./ThumbUtil');
+const StatsGenerator = require('./StatsGenerator');
const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(',');
const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz'];
+let statsLastSavedTime = null;
+
class Util {
static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER);
@@ -35,6 +28,10 @@ class Util {
return blockedExtensions.includes(extension);
}
+ static getMimeFromType(fileTypeMimeObj) {
+ return fileTypeMimeObj ? fileTypeMimeObj.mime : undefined;
+ }
+
static constructFilePublicLink(file) {
/*
TODO: This wont work without a reverse proxy serving both
@@ -102,7 +99,8 @@ class Util {
await db
.table('files')
.where('name', filename)
- .delete();
+ .delete()
+ .wasMutated();
}
} catch (error) {
log.error(`There was an error removing the file < ${filename} >`);
@@ -225,6 +223,7 @@ class Util {
static async storeFileToDb(req, res, user, file, db) {
const dbFile = await db.table('files')
+ // eslint-disable-next-line func-names
.where(function() {
if (user === undefined) {
this.whereNull('userId');
@@ -259,9 +258,9 @@ class Util {
let fileId;
if (process.env.DB_CLIENT === 'sqlite3') {
- fileId = await db.table('files').insert(data);
+ fileId = await db.table('files').insert(data).wasMutated();
} else {
- fileId = await db.table('files').insert(data, 'id');
+ fileId = await db.table('files').insert(data, 'id').wasMutated();
}
return {
@@ -275,7 +274,7 @@ class Util {
const now = moment.utc().toDate();
try {
- await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] });
+ await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }).wasMutated();
await db.table('albums').where('id', albumId).update('editedAt', now);
} catch (error) {
console.error(error);
@@ -311,6 +310,42 @@ class Util {
return extname + multi;
}
+
+ // TODO: Allow choosing what to save to db and what stats we care about in general
+ // TODO: if a stat is not saved to db but selected to be shows on the dashboard, it will be generated during the request
+ static async saveStatsToDb(force) {
+ // If there were no changes since the instance started, don't generate new stats
+ // OR
+ // if we alredy saved a stats to the db, and there were no new changes to the db since then
+ // skip generating and saving new stats.
+ if (!force &&
+ (!db.userParams.lastMutationTime ||
+ (statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime)
+ )
+ ) {
+ return;
+ }
+
+ const now = moment.utc().toDate();
+ const stats = await StatsGenerator.getStats(db);
+
+ let batchId = 1;
+
+ const res = (await db('statistics').max({ lastBatch: 'batchId' }))[0];
+ if (res && res.lastBatch) {
+ batchId = res.lastBatch + 1;
+ }
+
+ try {
+ for (const [type, data] of Object.entries(stats)) {
+ await db.table('statistics').insert({ type, data: JSON.stringify(data), createdAt: now, batchId });
+ }
+
+ statsLastSavedTime = now.getTime();
+ } catch (error) {
+ console.error(error);
+ }
+ }
}
module.exports = Util;
diff --git a/src/api/utils/multerStorage.js b/src/api/utils/multerStorage.js
index a2d01a4..1f1f0dd 100644
--- a/src/api/utils/multerStorage.js
+++ b/src/api/utils/multerStorage.js
@@ -2,13 +2,14 @@ const fs = require('fs');
const path = require('path');
const blake3 = require('blake3');
const jetpack = require('fs-jetpack');
+const FileType = require('file-type');
function DiskStorage(opts) {
this.getFilename = opts.filename;
if (typeof opts.destination === 'string') {
jetpack.dir(opts.destination);
- this.getDestination = function($0, $1, cb) { cb(null, opts.destination); };
+ this.getDestination = ($0, $1, cb) => { cb(null, opts.destination); };
} else {
this.getDestination = opts.destination;
}
@@ -52,25 +53,44 @@ DiskStorage.prototype._handleFile = function _handleFile(req, file, cb) {
file.stream.on('data', d => hash.update(d));
if (file._isChunk) {
- file.stream.on('end', () => {
- cb(null, {
- destination,
- filename,
- path: finalPath
+ if (file._chunksData.chunks === 0) {
+ FileType.stream(file.stream).then(ftStream => {
+ file._chunksData.fileType = ftStream.fileType;
+ file.stream.on('end', () => {
+ cb(null, {
+ destination,
+ filename,
+ path: finalPath,
+ fileType: file._chunksData.fileType
+ });
+ });
+ ftStream.pipe(outStream, { end: false });
});
- });
- file.stream.pipe(outStream, { end: false });
+ } else {
+ file.stream.on('end', () => {
+ cb(null, {
+ destination,
+ filename,
+ path: finalPath,
+ fileType: file._chunksData.fileType
+ });
+ });
+ file.stream.pipe(outStream, { end: false });
+ }
} else {
- outStream.on('finish', () => {
- cb(null, {
- destination,
- filename,
- path: finalPath,
- size: outStream.bytesWritten,
- hash: hash.digest('hex')
+ FileType.stream(file.stream).then(ftStream => {
+ outStream.on('finish', () => {
+ cb(null, {
+ destination,
+ filename,
+ path: finalPath,
+ size: outStream.bytesWritten,
+ hash: hash.digest('hex'),
+ fileType: ftStream.fileType
+ });
});
+ ftStream.pipe(outStream);
});
- file.stream.pipe(outStream);
}
});
});
@@ -86,6 +106,4 @@ DiskStorage.prototype._removeFile = function _removeFile(req, file, cb) {
fs.unlink(path, cb);
};
-module.exports = function(opts) {
- return new DiskStorage(opts);
-};
+module.exports = opts => new DiskStorage(opts);
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-21 13:03:30 +0000