Commit cd9a95fb authored by chaz's avatar chaz 👻

Write log of downloaded files

parent 9e8b13c3
......@@ -2,6 +2,7 @@
const archive = require('./archive.js');
const fs = require('fs');
const Log = require('./log.js');
const path = require('path');
const Queue = require('./queue.js');
const util_ = require('./util.js');
......@@ -55,6 +56,12 @@ const getArgs = () => {
requiresArg: true,
type: 'string',
},
log: {
alias: 'l',
defaultDescription: "based on collection, search, and the time",
description: "Log actions to this file (or use --no-log)",
type: 'string',
},
max: { // TODO: support max size to download.
coerce: coerceNonNegativeInt,
defaultDescription: "no maximum",
......@@ -120,8 +127,16 @@ const getArgs = () => {
.argv;
};
const shouldSkipFile
= async (file, filePath, progress, skipExisting = false) => {
const shouldSkipFile = async (
item,
file,
filePath,
{
log,
progress,
skipExisting = false,
} = {},
) => {
try {
await fs.promises.stat(filePath);
}
......@@ -143,10 +158,14 @@ const shouldSkipFile
const shouldSkip
= (await util_.hashFile(filePath, algorithm) === file[algorithm]);
if (shouldSkip)
if (shouldSkip) {
log(`${item}/${file.originalFilename}: verified checksum on ${filePath}`);
progress.setStatus(`Verified ${file.originalFilename}`);
else
}
else {
log(`${item}/${file.originalFilename}: failed checksum on ${filePath}`);
progress.setStatus(`Verification failed on ${file.originalFilename}`);
}
return shouldSkip;
};
......@@ -204,7 +223,7 @@ const main = module.exports = async () => {
return path.join(destination, bucket);
};
const getDownloadPath = async (item, file, fileCount, progress) => {
const getDownloadPath = async (item, file, fileCount, {log, progress}) => {
let dest = getBasePath(item);
if (fileCount > 1)
dest = path.join(dest, util_.sanitizeFilename(item));
......@@ -214,21 +233,46 @@ const main = module.exports = async () => {
: util_.sanitizeFilename(`${item}-${file.filename}`));
await fs.promises.mkdir(dest, {mode: 0o755, recursive: true});
if (await shouldSkipFile(
file, filePath, progress, args['skip-existing']))
if (await shouldSkipFile(item, file, filePath,
{log, progress, skipExisting: args['skip-existing']}))
return null;
return filePath;
}
let logPath = args.log;
if (logPath === undefined) {
const now = new Date().toISOString();
logPath = util_.sanitizeFilename(args.search !== undefined
? `${archive.URL.hostname}-${args.collection}-${args.search}.${now}.log`
: `${archive.URL.hostname}-${args.collection}.${now}.log`);
}
const log = (logPath ? new Log(logPath) : {
logEvent: () => {},
wait: () => {},
});
// TODO: log stuff like collection, search term, etc.?
say(`Archiving original files into ${destination} by ${args.sort}...`);
const q = new Queue(args.parallel, getDownloadPath, {quiet: args.quiet});
const q = new Queue(
args.parallel,
getDownloadPath,
{
log: message => { log.logEvent(message); },
onDownloaded: (item, file, path, {log}) => {
log(`${item}/${file.originalFilename}: downloaded ${path}`);
},
quiet: args.quiet,
},
);
q.addItems(items);
await q.wait();
q.close();
await log.wait();
process.removeListener('unhandledRejection', onUnhandledRejection);
};
......
module.exports = {
archive: require('./archive.js'),
Log: require('./log.js'),
main: require('./archive-collection.js'),
Progress: require('./progress.js'),
Queue: require('./queue.js'),
......
const fs = require('fs');
module.exports = class Log {
constructor(filePath) {
this.filePath = filePath;
this.writeLock = Promise.resolve();
this.events = [];
}
logEvent(message) {
const writeNow = !this.events.length;
this.events.push(message);
if (writeNow) {
this.writeLock = this.writeLock.then(() => {
const output = this.events.splice(0, this.events.length)
.join("\n") + "\n";
return fs.promises.appendFile(
this.filePath, output, {mode: 0o644});
});
}
}
wait() {
return this.writeLock;
}
};
......@@ -2,9 +2,19 @@ const archive = require('./archive.js');
const Progress = require('./progress.js');
module.exports = class Queue {
constructor(parallel, getDownloadPath, {quiet = false} = {}) {
constructor(
parallel,
getDownloadPath,
{
log = () => {},
onDownloaded = () => {},
quiet = false,
} = {},
) {
this.parallel = parallel;
this.getDownloadPath = getDownloadPath;
this.log = log;
this.onDownloaded = onDownloaded;
// Each task in taskQueue is a(n async) function with one argument, a
// Progress to keep updated.
......@@ -137,12 +147,13 @@ module.exports = class Queue {
progress.reset();
progress.setStatus(file.originalFilename, true);
const path
= await this.getDownloadPath(item, file, fileCount, progress);
const path = await this.getDownloadPath(
item, file, fileCount, {log: this.log, progress});
if (path) {
await archive.downloadFile(file, path, (downloaded, size) => {
progress.setRatio(downloaded, size);
});
this.onDownloaded(item, file, path, {log: this.log});
}
this.itemState[item].archivedFiles++;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment