Commit 236240bb authored by chaz's avatar chaz 👻

Support max and page args

parent b92f5c14
......@@ -64,10 +64,18 @@ const main = async () => {
requiresArg: true,
type: 'string',
},
limit: { // TODO: support max size to download.
max: { // TODO: support max size to download.
coerce: coerceNonNegativeInt,
defaultDescription: "all",
description: "How many items to download",
defaultDescription: "no maximum",
description: "The maximum number of items to download",
group: "Item selection:",
requiresArg: true,
type: 'number',
},
page: {
coerce: coerceNonNegativeInt,
default: 1, // In the actual search, 0 is equivalent to 1.
description: "max-sized (the --max arg) page number to download",
group: "Item selection:",
requiresArg: true,
type: 'number',
......@@ -112,14 +120,6 @@ const main = async () => {
requiresArg: true,
type: 'string',
},
start: {
coerce: coerceNonNegativeInt,
defaultDescription: "0",
description: "0-based item number at which to start downloading",
group: "Item selection:",
requiresArg: true,
type: 'number',
},
})
.help()
.alias('help', '?')
......@@ -145,9 +145,9 @@ const main = async () => {
say(`Listing items in ${args.collection}...`);
const items = await getItemsInCollection(args.collection, args.search, {
limit: args.limit,
max: args.max,
page: args.page,
sort: args.sort,
start: args.start,
});
say(`Archiving ${items.length} items`);
......@@ -286,12 +286,12 @@ const fetchJson = async (url) => {
return await response.json();
};
// TODO: support start and limit args, and @username (requires finding the
// email address of the username [looks like that requires scraping the html of
// the archive.org/details/@USERNAME page!] and then searching
// TODO: support @username (requires finding the email address of the username
// [looks like that requires scraping the html of the
// archive.org/details/@USERNAME page!] and then searching
// uploader:EMAIL_ADDRESS)!
const getItemsInCollection
= async (collection, search = '', {sort = 'createdate desc'} = {}) => {
const getItemsInCollection = async (collection, search = '',
{max = 0, page = 1, sort = 'createdate desc'} = {}) => {
const q = search
? `${search} AND collection:${collection}`
: `collection:${collection}`;
......@@ -302,21 +302,24 @@ const getItemsInCollection
callback: DUMMY_CALLBACK_NAME,
'fl[]': 'identifier',
output: 'json',
page: 1,
page,
q,
rows: 0,
rows: max,
'sort[]': sort,
});
url.search = params;
// TODO: batch rows by 10k, which I believe is the hard limit.
let data = await fetchJsonp(url);
// TODO: batch in some huge batch number? (10k I think is the hard limit.)
const count = parseInt(data.response.numFound, 10);
params.set('rows', count);
url.search = params;
if (!max) {
const count = parseInt(data.response.numFound, 10);
params.set('rows', count);
url.search = params;
data = await fetchJsonp(url);
}
data = await fetchJsonp(url);
return data.response.docs.map(d => String(d.identifier));
};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment