diff options
Diffstat (limited to 'node_modules/nugget')
| -rw-r--r-- | node_modules/nugget/.npmignore | 2 | ||||
| -rw-r--r-- | node_modules/nugget/.travis.yml | 3 | ||||
| -rwxr-xr-x | node_modules/nugget/bin.js | 32 | ||||
| -rw-r--r-- | node_modules/nugget/collaborators.md | 9 | ||||
| -rw-r--r-- | node_modules/nugget/index.js | 203 | ||||
| -rw-r--r-- | node_modules/nugget/multiple.png | bin | 0 -> 29002 bytes | |||
| -rw-r--r-- | node_modules/nugget/package.json | 66 | ||||
| -rw-r--r-- | node_modules/nugget/readme.md | 58 | ||||
| -rw-r--r-- | node_modules/nugget/test/cli.js | 11 | ||||
| -rw-r--r-- | node_modules/nugget/test/resume.html | 1 | ||||
| -rw-r--r-- | node_modules/nugget/test/resume.js | 41 | ||||
| -rw-r--r-- | node_modules/nugget/test/test.js | 38 | ||||
| -rw-r--r-- | node_modules/nugget/usage.txt | 8 |
13 files changed, 472 insertions, 0 deletions
diff --git a/node_modules/nugget/.npmignore b/node_modules/nugget/.npmignore new file mode 100644 index 0000000..91dfed8 --- /dev/null +++ b/node_modules/nugget/.npmignore @@ -0,0 +1,2 @@ +.DS_Store +node_modules
\ No newline at end of file diff --git a/node_modules/nugget/.travis.yml b/node_modules/nugget/.travis.yml new file mode 100644 index 0000000..244b7e8 --- /dev/null +++ b/node_modules/nugget/.travis.yml @@ -0,0 +1,3 @@ +language: node_js +node_js: + - '0.10' diff --git a/node_modules/nugget/bin.js b/node_modules/nugget/bin.js new file mode 100755 index 0000000..3f9c1c2 --- /dev/null +++ b/node_modules/nugget/bin.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node + +var fs = require('fs') +var path = require('path') +var nugget = require('./') +var args = require('minimist')(process.argv.slice(2)) + +var urls = args._ +if (urls.length === 0) { + console.log(fs.readFileSync(path.join(__dirname, 'usage.txt')).toString()) + process.exit(1) +} + +var opts = { + target: args.o || args.O || args.out, + dir: args.d || args.dir, + resume: args.c || args.continue, + force: args.f || args.force, + sockets: args.s || args.sockets, + quiet: args.q || args.quiet, + frequency: args.frequency ? +args.frequency : null, + proxy: args.proxy ? args.proxy : null, + strictSSL: args['strict-ssl'] +} + +nugget(urls, opts, function (err) { + if (err) { + console.error('Error:', err) + process.exit(1) + } + process.exit(0) +}) diff --git a/node_modules/nugget/collaborators.md b/node_modules/nugget/collaborators.md new file mode 100644 index 0000000..860d41f --- /dev/null +++ b/node_modules/nugget/collaborators.md @@ -0,0 +1,9 @@ +## Collaborators + +nugget is only possible due to the excellent work of the following collaborators: + +<table><tbody><tr><th align="left">maxogden</th><td><a href="https://github.com/maxogden">GitHub/maxogden</a></td></tr> +<tr><th align="left">grncdr</th><td><a href="https://github.com/grncdr">GitHub/grncdr</a></td></tr> +<tr><th align="left">mafintosh</th><td><a href="https://github.com/mafintosh">GitHub/mafintosh</a></td></tr> +<tr><th align="left">jlord</th><td><a href="https://github.com/jlord">GitHub/jlord</a></td></tr> +</tbody></table> diff --git a/node_modules/nugget/index.js b/node_modules/nugget/index.js new file mode 100644 index 0000000..b36918b --- /dev/null +++ b/node_modules/nugget/index.js @@ -0,0 +1,203 @@ +var request = require('request') +var fs = require('fs') +var path = require('path') +var log = require('single-line-log').stdout +var progress = require('progress-stream') +var prettyBytes = require('pretty-bytes') +var throttle = require('throttleit') +var EventEmitter = require('events').EventEmitter +var debug = require('debug')('nugget') + +function noop () {} + +module.exports = function (urls, opts, cb) { + if (!Array.isArray(urls)) urls = [urls] + if (urls.length === 1) opts.singleTarget = true + + var defaultProps = {} + + if (opts.sockets) { + var sockets = +opts.sockets + defaultProps.pool = {maxSockets: sockets} + } + + if (opts.proxy) { + defaultProps.proxy = opts.proxy + } + + if (opts.strictSSL !== null) { + defaultProps.strictSSL = opts.strictSSL + } + + if (Object.keys(defaultProps).length > 0) { + request = request.defaults(defaultProps) + } + + var downloads = [] + var errors = [] + var pending = 0 + var truncated = urls.length * 2 >= (process.stdout.rows - 15) + + urls.forEach(function (url) { + debug('start dl', url) + pending++ + var dl = startDownload(url, opts, function done (err) { + debug('done dl', url, pending) + if (err) { + debug('error dl', url, err) + errors.push(err) + dl.error = err.message + } + if (truncated) { + var i = downloads.indexOf(dl) + downloads.splice(i, 1) + downloads.push(dl) + } + if (--pending === 0) { + render() + cb(errors.length ? errors : undefined) + } + }) + + downloads.push(dl) + + dl.on('start', function (progressStream) { + throttledRender() + }) + + dl.on('progress', function (data) { + debug('progress', url, data.percentage) + + dl.speed = data.speed + if (dl.percentage === 100) render() + else throttledRender() + }) + }) + + var _log = opts.quiet ? noop : log + render() + var throttledRender = throttle(render, opts.frequency || 250) + + if (opts.singleTarget) return downloads[0] + else return downloads + + function render () { + var height = process.stdout.rows + var rendered = 0 + var output = '' + var totalSpeed = 0 + downloads.forEach(function (dl) { + if (2 * rendered >= height - 15) return + rendered++ + if (dl.error) { + output += 'Downloading ' + path.basename(dl.target) + '\n' + output += 'Error: ' + dl.error + '\n' + return + } + var pct = dl.percentage + var speed = dl.speed + var total = dl.fileSize + totalSpeed += speed + var bar = Array(Math.floor(45 * pct / 100)).join('=') + '>' + while (bar.length < 45) bar += ' ' + output += 'Downloading ' + path.basename(dl.target) + '\n' + + '[' + bar + '] ' + pct.toFixed(1) + '%' + if (total) output += ' of ' + prettyBytes(total) + output += ' (' + prettyBytes(speed) + '/s)\n' + }) + if (rendered < downloads.length) output += '\n... and ' + (downloads.length - rendered) + ' more\n' + if (downloads.length > 1) output += '\nCombined Speed: ' + prettyBytes(totalSpeed) + '/s\n' + _log(output) + } + + function startDownload (url, opts, cb) { + var targetName = path.basename(url).split('?')[0] + if (opts.singleTarget && opts.target) targetName = opts.target + var target = path.resolve(opts.dir || process.cwd(), targetName) + if (opts.resume) { + resume(url, opts, cb) + } else { + download(url, opts, cb) + } + + var progressEmitter = new EventEmitter() + progressEmitter.target = target + progressEmitter.speed = 0 + progressEmitter.percentage = 0 + + return progressEmitter + + function resume (url, opts, cb) { + fs.stat(target, function (err, stats) { + if (err && err.code === 'ENOENT') { + return download(url, opts, cb) + } + if (err) { + return cb(err) + } + var offset = stats.size + var req = request.get(url) + + req.on('error', cb) + req.on('response', function (resp) { + resp.destroy() + + var length = parseInt(resp.headers['content-length'], 10) + + // file is already downloaded. + if (length === offset) return cb() + + if (!isNaN(length) && length > offset && /bytes/.test(resp.headers['accept-ranges'])) { + opts.range = [offset, length] + } + + download(url, opts, cb) + }) + }) + } + + function download (url, opts, cb) { + var headers = opts.headers || {} + if (opts.range) { + headers.Range = 'bytes=' + opts.range[0] + '-' + opts.range[1] + } + var read = request(url, { headers: headers }) + + read.on('error', cb) + read.on('response', function (resp) { + debug('response', url, resp.statusCode) + if (resp.statusCode > 299 && !opts.force) return cb(new Error('GET ' + url + ' returned ' + resp.statusCode)) + var write = fs.createWriteStream(target, {flags: opts.resume ? 'a' : 'w'}) + write.on('error', cb) + write.on('finish', cb) + + var fullLen + var contentLen = Number(resp.headers['content-length']) + var range = resp.headers['content-range'] + if (range) { + fullLen = Number(range.split('/')[1]) + } else { + fullLen = contentLen + } + + progressEmitter.fileSize = fullLen + if (range) { + var downloaded = fullLen - contentLen + } + var progressStream = progress({ length: fullLen, transferred: downloaded }, onprogress) + progressEmitter.emit('start', progressStream) + + resp + .pipe(progressStream) + .pipe(write) + }) + + function onprogress (p) { + var pct = p.percentage + progressEmitter.progress = p + progressEmitter.percentage = pct + progressEmitter.emit('progress', p) + } + } + } +} diff --git a/node_modules/nugget/multiple.png b/node_modules/nugget/multiple.png Binary files differnew file mode 100644 index 0000000..67b7fca --- /dev/null +++ b/node_modules/nugget/multiple.png diff --git a/node_modules/nugget/package.json b/node_modules/nugget/package.json new file mode 100644 index 0000000..3c1a4cd --- /dev/null +++ b/node_modules/nugget/package.json @@ -0,0 +1,66 @@ +{ + "_from": "nugget@^2.0.0", + "_id": "[email protected]", + "_inBundle": false, + "_integrity": "sha1-IBCVpIfhrTYIGzQy+jytpPjQcbA=", + "_location": "/nugget", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "nugget@^2.0.0", + "name": "nugget", + "escapedName": "nugget", + "rawSpec": "^2.0.0", + "saveSpec": null, + "fetchSpec": "^2.0.0" + }, + "_requiredBy": [ + "/electron-download" + ], + "_resolved": "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz", + "_shasum": "201095a487e1ad36081b3432fa3cada4f8d071b0", + "_spec": "nugget@^2.0.0", + "_where": "/Users/armanshah/Desktop/node-projects/shopping-list/node_modules/electron-download", + "author": { + "name": "max ogden" + }, + "bin": { + "nugget": "bin.js" + }, + "bugs": { + "url": "https://github.com/maxogden/nugget/issues" + }, + "bundleDependencies": false, + "dependencies": { + "debug": "^2.1.3", + "minimist": "^1.1.0", + "pretty-bytes": "^1.0.2", + "progress-stream": "^1.1.0", + "request": "^2.45.0", + "single-line-log": "^1.1.2", + "throttleit": "0.0.2" + }, + "deprecated": false, + "description": "minimalist wget clone written in node. HTTP GETs a file and saves it to the current working directory", + "devDependencies": { + "standard": "^6.0.5", + "tape": "^3.0.1", + "tape-spawn": "^1.4.2" + }, + "directories": { + "test": "test" + }, + "homepage": "https://github.com/maxogden/nugget", + "license": "BSD", + "main": "index.js", + "name": "nugget", + "repository": { + "type": "git", + "url": "git://github.com/maxogden/nugget.git" + }, + "scripts": { + "test": "standard && tape test/*.js" + }, + "version": "2.0.1" +} diff --git a/node_modules/nugget/readme.md b/node_modules/nugget/readme.md new file mode 100644 index 0000000..439975b --- /dev/null +++ b/node_modules/nugget/readme.md @@ -0,0 +1,58 @@ +# nugget + +Minimalist command line downloader written in node, inspired by wget. HTTP GETs a file and streams it into a file in the current working directory. Specializes at downloading many files in parallel. + +[](https://nodei.co/npm/nugget/) + +[](https://travis-ci.org/maxogden/nugget) + +## installation + +``` +npm install nugget -g +``` + +## usage + +``` +Usage: nugget <urls> [options] + -o output filename + -d output parent directory + -c resume aborted download + -f ignore response codes > 299 + -s concurrent socket limit (default infinity) + -q disable logging +``` + +### examples + +``` +nugget http://foo.com/bar.jpg +# downloads bar.jpg and stores it in the current directory +``` + +or + +``` +nugget http://foo.com/bar.jpg -O baz.jpg +# saves it as baz.jpg. you can also do lowercase -o +``` + +if you get a statusCode of 300 or greater nugget will stop. you can force it to stream the response into a file anyway by doing `nugget http://404link.com/file.html -f` or `--force` works too + +you can also download multiple files, just pass multiple urls: + + + +## options + +The following options are recognized by nugget: + + * `-s|--sockets` - default Infinity. specify the number of http sockets to use at once (this controls concurrency) + * `-o|-O|--out` - specify the filename to write to. this only works if you are downloading a single file + * `-d|--dir` - save files in a directory other than the current one. + * `-c|--continue` - resume downloads if a partially complete target file already exists. If the target file exists and is the same size as the remote file, nothing will be done. + * `-f|--force` - force the server response to be saved to the target file, even if it's a non-successful status code. + * `-q|--quiet` - disable logging + * `--no-strict-ssl` - disable strict ssl + * `--proxy` - specify a proxy to use diff --git a/node_modules/nugget/test/cli.js b/node_modules/nugget/test/cli.js new file mode 100644 index 0000000..7d7313f --- /dev/null +++ b/node_modules/nugget/test/cli.js @@ -0,0 +1,11 @@ +var fs = require('fs') +var path = require('path') +var spawn = require('tape-spawn') +var test = require('tape') + +test('usage', function (t) { + var child = spawn(t, path.join(__dirname, '..', 'bin.js')) + child.stdout.match(fs.readFileSync(path.join(__dirname, '..', 'usage.txt')).toString() + '\n') + child.stderr.empty() + child.end() +}) diff --git a/node_modules/nugget/test/resume.html b/node_modules/nugget/test/resume.html new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/node_modules/nugget/test/resume.html @@ -0,0 +1 @@ +hello
\ No newline at end of file diff --git a/node_modules/nugget/test/resume.js b/node_modules/nugget/test/resume.js new file mode 100644 index 0000000..291ce0e --- /dev/null +++ b/node_modules/nugget/test/resume.js @@ -0,0 +1,41 @@ +var fs = require('fs') +var http = require('http') +var nugget = require('../') +var path = require('path') +var test = require('tape') + +var data = new Buffer('hello everybody I am the data') + +var testServer = http.createServer(function (req, res) { + if (!req.headers['range']) { + res.setHeader('content-length', data.length) + res.setHeader('accept-ranges', 'bytes') + res.end(data) + } else { + var range = req.headers['range'].split('=').pop().split('-').map(function (s) { + return parseInt(s, 10) + }) + res.setHeader('content-length', range[1] - range[0]) + res.setHeader('content-range', range[0] + '-' + range[1] + '/' + data.length) + res.end(data.slice(range[0], range[1])) + } +}) + +var target = path.join(__dirname, 'foobar.html') +if (fs.existsSync(target)) fs.unlinkSync(target) + +fs.writeFileSync(target, data.slice(0, 10)) + +testServer.listen(0, function () { + var port = this.address().port + test('fetches rest of file', function (t) { + nugget('http://localhost:' + port + '/foobar.html', {dir: __dirname, resume: true, quiet: true}, function (err) { + if (err) t.ifErr(err) + t.ok(fs.existsSync(target), 'downloaded file') + t.equal(fs.statSync(target).size, data.length, 'file is complete') + if (fs.existsSync(target)) fs.unlinkSync(target) + t.end() + testServer.close() + }) + }) +}) diff --git a/node_modules/nugget/test/test.js b/node_modules/nugget/test/test.js new file mode 100644 index 0000000..62ccb25 --- /dev/null +++ b/node_modules/nugget/test/test.js @@ -0,0 +1,38 @@ +var fs = require('fs') +var http = require('http') +var nugget = require('../') +var path = require('path') +var test = require('tape') + +var testServer = http.createServer(function (req, res) { + res.end('hello') +}) + +var target = path.join(__dirname, 'resume.html') +if (fs.existsSync(target)) fs.unlinkSync(target) + +testServer.listen(0, function () { + var port = this.address().port + test('fetches file', function (t) { + nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) { + if (err) t.ifErr(err) + t.ok(fs.existsSync(target), 'downloaded file') + if (fs.existsSync(target)) fs.unlinkSync(target) + t.end() + }) + }) + + test('has progress events', function (t) { + var gotProgress = false + var dl = nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) { + t.notOk(err, 'no error') + t.ok(gotProgress, 'got progress event') + t.end() + testServer.close() + }) + dl.once('progress', function (data) { + t.ok(data.hasOwnProperty('percentage'), 'has percentage') + gotProgress = true + }) + }) +}) diff --git a/node_modules/nugget/usage.txt b/node_modules/nugget/usage.txt new file mode 100644 index 0000000..90e2110 --- /dev/null +++ b/node_modules/nugget/usage.txt @@ -0,0 +1,8 @@ +Usage: nugget <urls> [options] + -o output filename + -d output parent directory + -c resume aborted download + -f ignore response codes > 299 + -s concurrent socket limit (default infinity) + -q disable logging + --stdin download all urls written to stdin (newline delimited)
\ No newline at end of file |
