aboutsummaryrefslogtreecommitdiff
path: root/src/api/utils
diff options
context:
space:
mode:
Diffstat (limited to 'src/api/utils')
-rw-r--r--src/api/utils/Util.js165
-rw-r--r--src/api/utils/multerStorage.js91
-rw-r--r--src/api/utils/rehashDatabase.js54
3 files changed, 237 insertions, 73 deletions
diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js
index c703ad5..d992d55 100644
--- a/src/api/utils/Util.js
+++ b/src/api/utils/Util.js
@@ -15,7 +15,6 @@ const db = require('knex')({
useNullAsDefault: process.env.DB_CLIENT === 'sqlite'
});
const moment = require('moment');
-const crypto = require('crypto');
const Zip = require('adm-zip');
const uuidv4 = require('uuid/v4');
@@ -23,6 +22,7 @@ const log = require('./Log');
const ThumbUtil = require('./ThumbUtil');
const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(',');
+const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz'];
class Util {
static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER);
@@ -50,12 +50,12 @@ class Util {
return file;
}
- static getUniqueFilename(name) {
+ static getUniqueFilename(extension) {
const retry = (i = 0) => {
const filename = randomstring.generate({
length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10),
capitalization: 'lowercase'
- }) + path.extname(name).toLowerCase();
+ }) + extension;
// TODO: Change this to look for the file in the db instead of in the filesystem
const exists = jetpack.exists(path.join(Util.uploadPath, filename));
@@ -88,37 +88,6 @@ class Util {
return retry();
}
- static async getFileHash(filename) {
- const file = await jetpack.readAsync(path.join(Util.uploadPath, filename), 'buffer');
- if (!file) {
- log.error(`There was an error reading the file < ${filename} > for hashing`);
- return null;
- }
-
- const hash = crypto.createHash('md5');
- hash.update(file, 'utf8');
- return hash.digest('hex');
- }
-
- static generateFileHash(data) {
- const hash = crypto
- .createHash('md5')
- .update(data)
- .digest('hex');
- return hash;
- }
-
- static async checkIfFileExists(db, user, hash) {
- const exists = await db.table('files')
- .where(function() { // eslint-disable-line func-names
- if (user) this.where('userId', user.id);
- else this.whereNull('userId');
- })
- .where({ hash })
- .first();
- return exists;
- }
-
static getFilenameFromPath(fullPath) {
return fullPath.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
}
@@ -237,47 +206,67 @@ class Util {
}
static generateThumbnails = ThumbUtil.generateThumbnails;
- static async saveFileToDatabase(req, res, user, db, file, originalFile) {
- /*
- Save the upload information to the database
- */
+
+ static async fileExists(res, exists, filename) {
+ exists = Util.constructFilePublicLink(exists);
+ res.json({
+ message: 'Successfully uploaded the file.',
+ name: exists.name,
+ hash: exists.hash,
+ size: exists.size,
+ url: `${process.env.DOMAIN}/${exists.name}`,
+ deleteUrl: `${process.env.DOMAIN}/api/file/${exists.id}`,
+ repeated: true
+ });
+
+ return this.deleteFile(filename);
+ }
+
+ static async storeFileToDb(req, res, user, file, db) {
+ const dbFile = await db.table('files')
+ .where(function() {
+ if (user === undefined) {
+ this.whereNull('userid');
+ } else {
+ this.where('userid', user.id);
+ }
+ })
+ .where({
+ hash: file.data.hash,
+ size: file.data.size
+ })
+ .first();
+
+ if (dbFile) {
+ await this.fileExists(res, dbFile, file.data.filename);
+ return;
+ }
+
const now = moment.utc().toDate();
- let insertedId = null;
- try {
- /*
- This is so fucking dumb
- */
- if (process.env.DB_CLIENT === 'sqlite3') {
- insertedId = await db.table('files').insert({
- userId: user ? user.id : null,
- name: file.name,
- original: originalFile.originalname,
- type: originalFile.mimetype || '',
- size: file.size,
- hash: file.hash,
- ip: req.ip,
- createdAt: now,
- editedAt: now
- });
- } else {
- insertedId = await db.table('files').insert({
- userId: user ? user.id : null,
- name: file.name,
- original: originalFile.originalname,
- type: originalFile.mimetype || '',
- size: file.size,
- hash: file.hash,
- ip: req.ip,
- createdAt: now,
- editedAt: now
- }, 'id');
- }
- return insertedId;
- } catch (error) {
- console.error('There was an error saving the file to the database');
- console.error(error);
- return null;
+ const data = {
+ userId: user ? user.id : null,
+ name: file.data.filename,
+ original: file.data.originalname,
+ type: file.data.mimetype,
+ size: file.data.size,
+ hash: file.data.hash,
+ ip: req.ip,
+ createdAt: now,
+ editedAt: now
+ };
+ Util.generateThumbnails(file.data.filename);
+
+ let fileId;
+ if (process.env.DB_CLIENT === 'sqlite3') {
+ fileId = await db.table('files').insert(data);
+ } else {
+ fileId = await db.table('files').insert(data, 'id');
}
+
+ return {
+ file: data,
+ id: fileId
+ };
}
static async saveFileToAlbum(db, albumId, insertedId) {
@@ -291,6 +280,36 @@ class Util {
console.error(error);
}
}
+
+ static getExtension(filename) {
+ // Always return blank string if the filename does not seem to have a valid extension
+ // Files such as .DS_Store (anything that starts with a dot, without any extension after) will still be accepted
+ if (!/\../.test(filename)) return '';
+
+ let lower = filename.toLowerCase(); // due to this, the returned extname will always be lower case
+ let multi = '';
+ let extname = '';
+
+ // check for multi-archive extensions (.001, .002, and so on)
+ if (/\.\d{3}$/.test(lower)) {
+ multi = lower.slice(lower.lastIndexOf('.') - lower.length);
+ lower = lower.slice(0, lower.lastIndexOf('.'));
+ }
+
+ // check against extensions that must be preserved
+ for (const extPreserve of preserveExtensions) {
+ if (lower.endsWith(extPreserve)) {
+ extname = extPreserve;
+ break;
+ }
+ }
+
+ if (!extname) {
+ extname = lower.slice(lower.lastIndexOf('.') - lower.length); // path.extname(lower)
+ }
+
+ return extname + multi;
+ }
}
module.exports = Util;
diff --git a/src/api/utils/multerStorage.js b/src/api/utils/multerStorage.js
new file mode 100644
index 0000000..a2d01a4
--- /dev/null
+++ b/src/api/utils/multerStorage.js
@@ -0,0 +1,91 @@
+const fs = require('fs');
+const path = require('path');
+const blake3 = require('blake3');
+const jetpack = require('fs-jetpack');
+
+function DiskStorage(opts) {
+ this.getFilename = opts.filename;
+
+ if (typeof opts.destination === 'string') {
+ jetpack.dir(opts.destination);
+ this.getDestination = function($0, $1, cb) { cb(null, opts.destination); };
+ } else {
+ this.getDestination = opts.destination;
+ }
+}
+
+DiskStorage.prototype._handleFile = function _handleFile(req, file, cb) {
+ const that = this; // eslint-disable-line consistent-this
+
+ that.getDestination(req, file, (err, destination) => {
+ if (err) return cb(err);
+
+ that.getFilename(req, file, (err, filename) => {
+ if (err) return cb(err);
+
+ const finalPath = path.join(destination, filename);
+ const onerror = err => {
+ hash.dispose(); // eslint-disable-line no-use-before-define
+ cb(err);
+ };
+
+ let outStream;
+ let hash;
+ if (file._isChunk) {
+ if (!file._chunksData.stream) {
+ file._chunksData.stream = fs.createWriteStream(finalPath, { flags: 'a' });
+ file._chunksData.stream.on('error', onerror);
+ }
+ if (!file._chunksData.hasher) {
+ file._chunksData.hasher = blake3.createHash();
+ }
+
+ outStream = file._chunksData.stream;
+ hash = file._chunksData.hasher;
+ } else {
+ outStream = fs.createWriteStream(finalPath);
+ outStream.on('error', onerror);
+ hash = blake3.createHash();
+ }
+
+ file.stream.on('error', onerror);
+ file.stream.on('data', d => hash.update(d));
+
+ if (file._isChunk) {
+ file.stream.on('end', () => {
+ cb(null, {
+ destination,
+ filename,
+ path: finalPath
+ });
+ });
+ file.stream.pipe(outStream, { end: false });
+ } else {
+ outStream.on('finish', () => {
+ cb(null, {
+ destination,
+ filename,
+ path: finalPath,
+ size: outStream.bytesWritten,
+ hash: hash.digest('hex')
+ });
+ });
+ file.stream.pipe(outStream);
+ }
+ });
+ });
+};
+
+DiskStorage.prototype._removeFile = function _removeFile(req, file, cb) {
+ const path = file.path;
+
+ delete file.destination;
+ delete file.filename;
+ delete file.path;
+
+ fs.unlink(path, cb);
+};
+
+module.exports = function(opts) {
+ return new DiskStorage(opts);
+};
diff --git a/src/api/utils/rehashDatabase.js b/src/api/utils/rehashDatabase.js
new file mode 100644
index 0000000..4ccb301
--- /dev/null
+++ b/src/api/utils/rehashDatabase.js
@@ -0,0 +1,54 @@
+require('dotenv').config();
+const blake3 = require('blake3');
+const path = require('path');
+const fs = require('fs');
+const db = require('knex')({
+ client: 'sqlite3',
+ connection: {
+ filename: path.join(__dirname, '../../../database/', 'database.sqlite')
+ }
+});
+
+const start = async () => {
+ const hrstart = process.hrtime();
+ const uploads = await db.table('files')
+ .select('id', 'name', 'hash');
+ console.log(`Uploads : ${uploads.length}`);
+
+
+ let done = 0;
+ const printProgress = () => {
+ console.log(`PROGRESS: ${done}/${uploads.length}`);
+ if (done >= uploads.length) clearInterval(progressInterval);
+ };
+ const progressInterval = setInterval(printProgress, 1000);
+ printProgress();
+
+ for (const upload of uploads) {
+ await new Promise((resolve, reject) => {
+ fs.createReadStream(path.join(__dirname, '../../../uploads', upload.name))
+ .on('error', reject)
+ .pipe(blake3.createHash())
+ .on('error', reject)
+ .on('data', async source => {
+ const hash = source.toString('hex');
+ console.log(`${upload.name}: ${hash}`);
+ await db.table('files')
+ .update('hash', hash)
+ .where('id', upload.id);
+ done++;
+ resolve();
+ });
+ }).catch(error => {
+ console.log(`${upload.name}: ${error.toString()}`);
+ });
+ }
+
+ clearInterval(progressInterval);
+ printProgress();
+
+ const hrend = process.hrtime(hrstart);
+ console.log(`Done in : ${(hrend[0] + (hrend[1] / 1e9)).toFixed(4)}s`);
+};
+
+start();
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-28 01:31:18 +0000