Write log of downloaded files

archive-collection.js

@@ -2,6 +2,7 @@
 
 const archive = require('./archive.js');
 const fs = require('fs');
+const Log = require('./log.js');
 const path = require('path');
 const Queue = require('./queue.js');
 const util_ = require('./util.js');
@@ -55,6 +56,12 @@ const getArgs = () => {
             requiresArg: true,
             type: 'string',
         },
+        log: {
+            alias: 'l',
+            defaultDescription: "based on collection, search, and the time",
+            description: "Log actions to this file (or use --no-log)",
+            type: 'string',
+        },
         max: {  // TODO: support max size to download.
             coerce: coerceNonNegativeInt,
             defaultDescription: "no maximum",
@@ -120,8 +127,16 @@ const getArgs = () => {
     .argv;
 };
 
-const shouldSkipFile
-        = async (file, filePath, progress, skipExisting = false) => {
+const shouldSkipFile = async (
+    item,
+    file,
+    filePath,
+    {
+        log,
+        progress,
+        skipExisting = false,
+    } = {},
+) => {
     try {
         await fs.promises.stat(filePath);
     }
@@ -143,10 +158,14 @@ const shouldSkipFile
     const shouldSkip
             = (await util_.hashFile(filePath, algorithm) === file[algorithm]);
 
-    if (shouldSkip)
+    if (shouldSkip) {
+        log(`${item}/${file.originalFilename}: verified checksum on ${filePath}`);
         progress.setStatus(`Verified ${file.originalFilename}`);
-    else
+    }
+    else {
+        log(`${item}/${file.originalFilename}: failed checksum on ${filePath}`);
         progress.setStatus(`Verification failed on ${file.originalFilename}`);
+    }
 
     return shouldSkip;
 };
@@ -204,7 +223,7 @@ const main = module.exports = async () => {
         return path.join(destination, bucket);
     };
 
-    const getDownloadPath = async (item, file, fileCount, progress) => {
+    const getDownloadPath = async (item, file, fileCount, {log, progress}) => {
         let dest = getBasePath(item);
         if (fileCount > 1)
             dest = path.join(dest, util_.sanitizeFilename(item));
@@ -214,21 +233,46 @@ const main = module.exports = async () => {
                 : util_.sanitizeFilename(`${item}-${file.filename}`));
 
         await fs.promises.mkdir(dest, {mode: 0o755, recursive: true});
-        if (await shouldSkipFile(
-                file, filePath, progress, args['skip-existing']))
+        if (await shouldSkipFile(item, file, filePath,
+                {log, progress, skipExisting: args['skip-existing']}))
             return null;
 
         return filePath;
     }
 
+    let logPath = args.log;
+    if (logPath === undefined) {
+        const now = new Date().toISOString();
+        logPath = util_.sanitizeFilename(args.search !== undefined
+            ? `${archive.URL.hostname}-${args.collection}-${args.search}.${now}.log`
+            : `${archive.URL.hostname}-${args.collection}.${now}.log`);
+    }
+    const log = (logPath ? new Log(logPath) : {
+        logEvent: () => {},
+        wait: () => {},
+    });
+    // TODO: log stuff like collection, search term, etc.?
+
     say(`Archiving original files into ${destination} by ${args.sort}...`);
 
-    const q = new Queue(args.parallel, getDownloadPath, {quiet: args.quiet});
+    const q = new Queue(
+        args.parallel,
+        getDownloadPath,
+        {
+            log: message => { log.logEvent(message); },
+            onDownloaded: (item, file, path, {log}) => {
+                log(`${item}/${file.originalFilename}: downloaded ${path}`);
+            },
+            quiet: args.quiet,
+        },
+    );
     q.addItems(items);
 
     await q.wait();
-
     q.close();
+
+    await log.wait();
+
     process.removeListener('unhandledRejection', onUnhandledRejection);
 };
 

index.js

 module.exports = {
     archive: require('./archive.js'),
+    Log: require('./log.js'),
     main: require('./archive-collection.js'),
     Progress: require('./progress.js'),
     Queue: require('./queue.js'),

log.js

+const fs = require('fs');
+
+module.exports = class Log {
+    constructor(filePath) {
+        this.filePath = filePath;
+        this.writeLock = Promise.resolve();
+        this.events = [];
+    }
+
+    logEvent(message) {
+        const writeNow = !this.events.length;
+        this.events.push(message);
+
+        if (writeNow) {
+            this.writeLock = this.writeLock.then(() => {
+                const output = this.events.splice(0, this.events.length)
+                        .join("\n") + "\n";
+                return fs.promises.appendFile(
+                        this.filePath, output, {mode: 0o644});
+            });
+        }
+    }
+
+    wait() {
+        return this.writeLock;
+    }
+};

queue.js

@@ -2,9 +2,19 @@ const archive = require('./archive.js');
 const Progress = require('./progress.js');
 
 module.exports = class Queue {
-    constructor(parallel, getDownloadPath, {quiet = false} = {}) {
+    constructor(
+        parallel,
+        getDownloadPath,
+        {
+            log = () => {},
+            onDownloaded = () => {},
+            quiet = false,
+        } = {},
+    ) {
         this.parallel = parallel;
         this.getDownloadPath = getDownloadPath;
+        this.log = log;
+        this.onDownloaded = onDownloaded;
 
         // Each task in taskQueue is a(n async) function with one argument, a
         // Progress to keep updated.
@@ -137,12 +147,13 @@ module.exports = class Queue {
         progress.reset();
         progress.setStatus(file.originalFilename, true);
 
-        const path
-                = await this.getDownloadPath(item, file, fileCount, progress);
+        const path = await this.getDownloadPath(
+                item, file, fileCount, {log: this.log, progress});
         if (path) {
             await archive.downloadFile(file, path, (downloaded, size) => {
                 progress.setRatio(downloaded, size);
             });
+            this.onDownloaded(item, file, path, {log: this.log});
         }
 
         this.itemState[item].archivedFiles++;