Commit 12734422 authored by chaz's avatar chaz 👻

Only download files if hashes differ, by default

parent 6e5e5c7b
#!/usr/bin/env node
const crypto = require('crypto');
const fetch = require('node-fetch');
const fs = require('fs');
const path = require('path');
......@@ -157,7 +158,38 @@ const main = async () => {
return;
}
// TODO: hash file, skip if hashes match.
let algorithm;
if (file.sha1) algorithm = 'sha1';
else if (file.md5) algorithm = 'md5';
// Just re-download if the server didn't respond with any hashes.
if (algorithm) {
const hash = await new Promise((resolve, reject) => {
let hashBuf = Buffer.alloc(0);
const hasher = crypto.createHash(algorithm);
hasher.on('error', e => { reject(e); });
hasher.on('data', data => {
hashBuf = Buffer.concat([hashBuf, data]);
});
hasher.on('finish', () => {
resolve(hashBuf.toString('hex'));
});
const readStream = fs.createReadStream(path_);
readStream.on('error', e => { reject(e); });
readStream.pipe(hasher);
});
if (hash === file[algorithm]) {
say(` File ${path_} matches archive.org version, skipping`);
say("");
return;
}
else {
say(` File ${path_} differs from archive.org version, re-downloading`);
}
}
}
say(` Downloading ${file.url} > ${path_}...`);
......@@ -197,7 +229,10 @@ const fetchJson = async (url) => {
return await response.json();
};
// TODO: support search, start, and limit args, and @username!
// TODO: support search, start, and limit args, and @username (requires finding
// the email address of the username [looks like that requires scraping the
// html of the archive.org/details/@USERNAME page!] and then searching
// uploader:EMAIL_ADDRESS)!
const getItemsInCollection
= async (collection, {sort = 'createdate desc'} = {}) => {
const url = new URL(ARCHIVE_URL);
......@@ -266,7 +301,9 @@ const selectFileForItem = async (item, selector = selectBiggest) => {
url.pathname = `/download/${item}/${file.name}`;
return {
filename: sanitizeFilename(`${item}-${String(file.name)}`),
filename: sanitizeFilename(`${item}-${file.name}`),
md5: String(file.md5),
sha1: String(file.sha1),
url,
};
};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment