Updated mindgeek scraper for entities. Various fixes.

This commit is contained in:
2020-06-28 22:29:18 +02:00
parent 41d7d2fa34
commit 98c19b560f
32 changed files with 24 additions and 12 deletions

View File

@@ -46,10 +46,10 @@ const { argv } = yargs
type: 'boolean',
default: true,
})
.option('profiles', {
.option('scene-actors', {
describe: 'Scrape profiles for new actors after fetching scenes',
type: 'boolean',
alias: 'with-actors',
alias: 'with-profiles',
default: false,
})
.option('scene', {

View File

@@ -43,7 +43,7 @@ function scrapeLatestX(data, site) {
description: data.description,
};
const hostname = site.parameters?.native ? site.url : site.network.url;
const hostname = site.parameters?.native ? site.url : site.parent.url;
release.url = `${hostname}/scene/${release.entryId}/`;
release.date = new Date(data.dateReleased);
@@ -121,7 +121,7 @@ function getUrl(site) {
}
if (site.parameters?.siteId) {
return `${site.network.url}/scenes?site=${site.parameters.siteId}`;
return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
}
throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);

View File

@@ -236,7 +236,7 @@ async function storeReleases(releases) {
// media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId);
if (argv.withActors) {
if (argv.sceneActors) {
await scrapeActors(actors.map(actor => actor.name));
}