I am attempting to write a Node.js script that takes in a URL (to download a file) specified through the command line. Using the HTTP Range request header the script downloads the file in a configurable number of chunks and chunk size and then writes to an output file in the correct Byte order.
Currently looking to achieve this with 2 chunks of 1 MiB (1,048,576 Bytes) for a total of 2 MiB (2,097,152 B).
Current issue I am running into either in execution or just my understanding is that my script is writing ~ 1,800,000 Bytes for each request resulting in a total of 3,764,942 Bytes. Not sure where those extra Bytes are coming from?
Is this due to a mistake I missed in this script or overhead from the request library used maybe or am I missing something about Mib to Bytes conversion?
- Contents of file are largely irrelevant right now so long as correct number of Bytes are in correct order.
- Curling the test URL with the range header set to 1MiB chunks and then appending to a file results in closer to expected number of total Bytes.
curl 'https://eloquentjavascript.net/Eloquent_JavaScript.pdf' -i -H "Range: bytes=0-2097152"=> 2097435 B file - I am running this command in terminal
node index.js --url='https://eloquentjavascript.net/Eloquent_JavaScript.pdf' --file='newfile.txt' --chunks 2 - Using Node v10.12.0, minimist v1.2.0, and request-promise v4.2.2
Entire script below:
'use strict';
const argv = require('minimist')(process.argv.slice(2), {
default: {
file: 'output.txt',
MiB: 1,
chunks: 4
}
});
const fs = require('fs');
const request = require('request-promise');
// Source URL must be specified through command line option.
if (!argv.url) throw Error('Source URL is required!');
const options = {
method: 'GET',
uri: argv.url
}
const determineChunkRange = (step) => {
// 1 Mib = 1,048,576 B.
// Only 1 MiB chunks are downloaded.
const chunkSize = argv.MiB * 1048576;
const startOfRange = step === 0 ? 0 + ((chunkSize * step)) : 1 + ((chunkSize * step));
const endOfRange = startOfRange + chunkSize;
return {'Range': `bytes=${startOfRange}-${endOfRange}`}
}
const getOptions = (step) => {
options.headers = determineChunkRange(step);
return options;
}
const addDataToFile = (data) => {
try {
fs.appendFileSync(argv.file, data);
console.log("Data written to file.");
} catch (err) {
console.log(`Error appending to ${argv.file}`, err);
}
}
// Create or Replace file with specific filename.
fs.writeFileSync(argv.file, '');
console.log("Successfully created new file.");
// Make specified number of requests.
for (let i = 0; i < argv.chunks; i++) {
const options = getOptions(i);
// make request to specified URL.
request(options)
.then(response => {
console.log(i, options)
addDataToFile(response)
})
.catch(error => {
console.log(`Error making request to ${argv.url}`, error)
});
}
