feature: new chunk and write strategy

2 files changed, 186 insertions, 2 deletions

diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js
index c703ad5..76ba171 100644
--- a/src/api/utils/Util.js
+++ b/src/api/utils/Util.js
@@ -23,6 +23,7 @@ const log = require('./Log');
 const ThumbUtil = require('./ThumbUtil');
 
 const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(',');
+const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz'];
 
 class Util {
 	static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER);
@@ -50,12 +51,12 @@ class Util {
 		return file;
 	}
 
-	static getUniqueFilename(name) {
+	static getUniqueFilename(extension) {
 		const retry = (i = 0) => {
 			const filename = randomstring.generate({
 				length: parseInt(process.env.GENERATED_FILENAME_LENGTH, 10),
 				capitalization: 'lowercase'
-			}) + path.extname(name).toLowerCase();
+			}) + extension;
 
 			// TODO: Change this to look for the file in the db instead of in the filesystem
 			const exists = jetpack.exists(path.join(Util.uploadPath, filename));
@@ -280,6 +281,68 @@ class Util {
 		}
 	}
 
+	static async fileExists(res, exists, filename) {
+		exists = Util.constructFilePublicLink(exists);
+		res.json({
+			message: 'Successfully uploaded the file.',
+			name: exists.name,
+			hash: exists.hash,
+			size: exists.size,
+			url: `${process.env.DOMAIN}/${exists.name}`,
+			deleteUrl: `${process.env.DOMAIN}/api/file/${exists.id}`,
+			repeated: true
+		});
+
+		return this.deleteFile(filename);
+	}
+
+	static async storeFileToDb(req, res, user, file, db) {
+		const dbFile = await db.table('files')
+			.where(function() {
+				if (user === undefined) {
+					this.whereNull('userid');
+				} else {
+					this.where('userid', user.id);
+				}
+			})
+			.where({
+				hash: file.data.hash,
+				size: file.data.size
+			})
+			.first();
+
+		if (dbFile) {
+			await this.fileExists(res, dbFile, file.data.filename);
+			return;
+		}
+
+		const now = moment.utc().toDate();
+		const data = {
+			userId: user ? user.id : null,
+			name: file.data.filename,
+			original: file.data.originalname,
+			type: file.data.mimetype,
+			size: file.data.size,
+			hash: file.data.hash,
+			ip: req.ip,
+			createdAt: now,
+			editedAt: now
+		};
+		Util.generateThumbnails(file.data.filename);
+
+		let fileId;
+		if (process.env.DB_CLIENT === 'sqlite3') {
+			fileId = await db.table('files').insert(data);
+		} else {
+			fileId = await db.table('files').insert(data, 'id');
+		}
+
+		return {
+			file: data,
+			id: fileId
+		};
+	}
+
 	static async saveFileToAlbum(db, albumId, insertedId) {
 		if (!albumId) return;
 
@@ -291,6 +354,36 @@ class Util {
 			console.error(error);
 		}
 	}
+
+	static getExtension(filename) {
+		// Always return blank string if the filename does not seem to have a valid extension
+		// Files such as .DS_Store (anything that starts with a dot, without any extension after) will still be accepted
+		if (!/\../.test(filename)) return '';
+
+		let lower = filename.toLowerCase(); // due to this, the returned extname will always be lower case
+		let multi = '';
+		let extname = '';
+
+		// check for multi-archive extensions (.001, .002, and so on)
+		if (/\.\d{3}$/.test(lower)) {
+			multi = lower.slice(lower.lastIndexOf('.') - lower.length);
+			lower = lower.slice(0, lower.lastIndexOf('.'));
+		}
+
+		// check against extensions that must be preserved
+		for (const extPreserve of preserveExtensions) {
+			if (lower.endsWith(extPreserve)) {
+				extname = extPreserve;
+				break;
+			}
+		}
+
+		if (!extname) {
+			extname = lower.slice(lower.lastIndexOf('.') - lower.length); // path.extname(lower)
+		}
+
+		return extname + multi;
+	}
 }
 
 module.exports = Util;
diff --git a/src/api/utils/multerStorage.js b/src/api/utils/multerStorage.js
new file mode 100644
index 0000000..a2d01a4
--- /dev/null
+++ b/src/api/utils/multerStorage.js
@@ -0,0 +1,91 @@
+const fs = require('fs');
+const path = require('path');
+const blake3 = require('blake3');
+const jetpack = require('fs-jetpack');
+
+function DiskStorage(opts) {
+	this.getFilename = opts.filename;
+
+	if (typeof opts.destination === 'string') {
+		jetpack.dir(opts.destination);
+		this.getDestination = function($0, $1, cb) { cb(null, opts.destination); };
+	} else {
+		this.getDestination = opts.destination;
+	}
+}
+
+DiskStorage.prototype._handleFile = function _handleFile(req, file, cb) {
+	const that = this; // eslint-disable-line consistent-this
+
+	that.getDestination(req, file, (err, destination) => {
+		if (err) return cb(err);
+
+		that.getFilename(req, file, (err, filename) => {
+			if (err) return cb(err);
+
+			const finalPath = path.join(destination, filename);
+			const onerror = err => {
+				hash.dispose(); // eslint-disable-line no-use-before-define
+				cb(err);
+			};
+
+			let outStream;
+			let hash;
+			if (file._isChunk) {
+				if (!file._chunksData.stream) {
+					file._chunksData.stream = fs.createWriteStream(finalPath, { flags: 'a' });
+					file._chunksData.stream.on('error', onerror);
+				}
+				if (!file._chunksData.hasher) {
+					file._chunksData.hasher = blake3.createHash();
+				}
+
+				outStream = file._chunksData.stream;
+				hash = file._chunksData.hasher;
+			} else {
+				outStream = fs.createWriteStream(finalPath);
+				outStream.on('error', onerror);
+				hash = blake3.createHash();
+			}
+
+			file.stream.on('error', onerror);
+			file.stream.on('data', d => hash.update(d));
+
+			if (file._isChunk) {
+				file.stream.on('end', () => {
+					cb(null, {
+						destination,
+						filename,
+						path: finalPath
+					});
+				});
+				file.stream.pipe(outStream, { end: false });
+			} else {
+				outStream.on('finish', () => {
+					cb(null, {
+						destination,
+						filename,
+						path: finalPath,
+						size: outStream.bytesWritten,
+						hash: hash.digest('hex')
+					});
+				});
+				file.stream.pipe(outStream);
+			}
+		});
+	});
+};
+
+DiskStorage.prototype._removeFile = function _removeFile(req, file, cb) {
+	const path = file.path;
+
+	delete file.destination;
+	delete file.filename;
+	delete file.path;
+
+	fs.unlink(path, cb);
+};
+
+module.exports = function(opts) {
+	return new DiskStorage(opts);
+};