DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Improved scraping and association behavior.

This commit is contained in:
ThePendulum 2019-11-19 04:36:15 +01:00
parent 7e9fd19c2a
commit 3ec7b15886
10 changed files with 166 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
assets/components/actor/actor.vue
View File

@ -20,6 +20,11 @@
<span>{{ actor.aliases.join(', ') }}</span> <span>{{ actor.aliases.join(', ') }}</span>
</li> </li>
<li v-if="actor.gender">
<dfn class="bio-heading">Gender</dfn>
<span>{{ actor.gender }}</span>
</li>
<li v-if="actor.birthdate"> <li v-if="actor.birthdate">
<dfn class="bio-heading">Date of birth</dfn> <dfn class="bio-heading">Date of birth</dfn>
<span>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }} ({{ age }})</span> <span>{{ formatDate(actor.birthdate, 'MMMM D, YYYY') }} ({{ age }})</span>
@ -51,7 +56,7 @@
<span v-if="actor.residencePlace">{{ actor.residencePlace }}</span> <span v-if="actor.residencePlace">{{ actor.residencePlace }}</span>
</li> </li>
<li v-if="actor.caucasion"> <li v-if="actor.ethnicity">
<dfn class="bio-heading">Ethnicity</dfn> <dfn class="bio-heading">Ethnicity</dfn>
<span>{{ actor.ethnicity }}</span> <span>{{ actor.ethnicity }}</span>
</li> </li>
@ -61,7 +66,7 @@
<span>{{ actor.height }}</span> <span>{{ actor.height }}</span>
</li> </li>
<li v-if="actor.gender !== 'male'"> <li v-if="actor.boobSize || actor.boobsNatural">
<dfn class="bio-heading">Boobs</dfn> <dfn class="bio-heading">Boobs</dfn>
<span v-if="actor.boobSize">{{ actor.boobSize }}</span> <span v-if="actor.boobSize">{{ actor.boobSize }}</span>
<span v-if="actor.boobsNatural !== null">{{ actor.boobsNatural ? 'Natural' : 'Enhanced' }}</span> <span v-if="actor.boobsNatural !== null">{{ actor.boobsNatural ? 'Natural' : 'Enhanced' }}</span>

10
migrations/20190325001339_releases.js
View File

@ -53,6 +53,9 @@ exports.up = knex => Promise.resolve()
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
table.datetime('scraped_at');
table.boolean('scrape_success');
})) }))
.then(() => knex.schema.createTable('directors', (table) => { .then(() => knex.schema.createTable('directors', (table) => {
table.increments('id', 12); table.increments('id', 12);
@ -229,6 +232,8 @@ exports.up = knex => Promise.resolve()
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors');
table.unique(['release_id', 'actor_id']);
})) }))
.then(() => knex.schema.createTable('directors_associated', (table) => { .then(() => knex.schema.createTable('directors_associated', (table) => {
table.increments('id', 16); table.increments('id', 16);
@ -242,6 +247,8 @@ exports.up = knex => Promise.resolve()
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('directors'); .inTable('directors');
table.unique(['release_id', 'director_id']);
})) }))
.then(() => knex.schema.createTable('tags_associated', (table) => { .then(() => knex.schema.createTable('tags_associated', (table) => {
table.integer('tag_id', 12) table.integer('tag_id', 12)
@ -256,6 +263,9 @@ exports.up = knex => Promise.resolve()
table.integer('release_id', 16) table.integer('release_id', 16)
.references('id') .references('id')
.inTable('releases'); .inTable('releases');
table.unique(['release_id', 'tag_id']);
table.unique(['site_id', 'tag_id']);
})); }));
exports.down = knex => Promise.resolve() exports.down = knex => Promise.resolve()

78
src/actors.js
View File

@ -12,6 +12,7 @@ async function curateActor(actor) {
return { return {
id: actor.id, id: actor.id,
gender: actor.gender,
name: actor.name, name: actor.name,
description: actor.description, description: actor.description,
birthdate: actor.birthdate && new Date(actor.birthdate), birthdate: actor.birthdate && new Date(actor.birthdate),
@ -43,10 +44,13 @@ function curateActors(releases) {
return Promise.all(releases.map(async release => curateActor(release))); return Promise.all(releases.map(async release => curateActor(release)));
} }
function curateScrapedActor(actor) { function curateActorEntry(actor, scraped, scrapeSuccess) {
return { const curatedActor = {
id: actor.id, id: actor.id,
name: actor.name, name: actor.name
.split(' ')
.map(segment => `${segment.charAt(0).toUpperCase()}${segment.slice(1)}`)
.join(' '),
slug: actor.name.toLowerCase().replace(/\s+/g, '-'), slug: actor.name.toLowerCase().replace(/\s+/g, '-'),
birthdate: actor.birthdate, birthdate: actor.birthdate,
description: actor.description, description: actor.description,
@ -65,6 +69,16 @@ function curateScrapedActor(actor) {
tattoos: actor.tattoos, tattoos: actor.tattoos,
piercings: actor.piercings, piercings: actor.piercings,
}; };
if (scraped) {
return {
...curatedActor,
scraped_at: new Date(),
scrape_success: scrapeSuccess,
};
}
return curatedActor;
} }
async function fetchActors(queryObject) { async function fetchActors(queryObject) {
@ -82,8 +96,8 @@ async function fetchActors(queryObject) {
return curateActors(releases); return curateActors(releases);
} }
async function storeActor(actor) { async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateScrapedActor(actor); const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const actorEntries = await knex('actors') const actorEntries = await knex('actors')
.insert(curatedActor) .insert(curatedActor)
@ -102,8 +116,8 @@ async function storeActor(actor) {
return null; return null;
} }
async function updateActor(actorEntry, actor) { async function updateActor(actorEntry, actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateScrapedActor(actor); const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
const actorEntries = await knex('actors') const actorEntries = await knex('actors')
.where({ id: actorEntry.id }) .where({ id: actorEntry.id })
@ -117,39 +131,59 @@ async function updateActor(actorEntry, actor) {
async function scrapeActors(actorNames) { async function scrapeActors(actorNames) {
await Promise.map(actorNames || argv.actors, async (actorName) => { await Promise.map(actorNames || argv.actors, async (actorName) => {
const [actorEntry] = await fetchActors({ name: actorName }); const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const profiles = await Promise.all(Object.values(scrapers.actors).map(scraper => scraper.fetchActor(actorName)));
if (actorEntry) { const [actorEntry] = await fetchActors({ slug: actorSlug });
return updateActor(actorEntry, profiles[0]); const profiles = await Promise.all(Object.values(scrapers.actors).map(scraper => scraper.fetchActor(actorEntry ? actorEntry.name : actorName)));
if (profiles[0] === null) {
console.log(`Could not find profile for actor '${actorName}'`);
return updateActor(actorEntry, actorEntry, true, false);
} }
return storeActor(profiles[0]); if (actorEntry && profiles[0]) {
return updateActor(actorEntry, profiles[0], true, true);
}
return storeActor(profiles[0], true, true);
}, { }, {
concurrency: 5, concurrency: 1,
}); });
} }
async function storeActors(release, releaseEntry) { async function scrapeBasicActors() {
const basicActors = await knex('actors').where('scraped_at', null);
return scrapeActors(basicActors.map(actor => actor.name));
}
async function associateActors(release, releaseId) {
const actorEntries = await knex('actors').whereIn('name', release.actors); const actorEntries = await knex('actors').whereIn('name', release.actors);
const newActors = release.actors const newActors = release.actors
.map(actorName => actorName.trim()) .map(actorName => actorName.trim())
.filter(actorName => !actorEntries.some(actor => actor.name === actorName)); .filter(actorName => !actorEntries.some(actor => actor.name === actorName));
const newActorEntries = await Promise.all(newActors.map(async actorName => storeActor({ name: actorName }))); const [newActorEntries, associatedActors] = await Promise.all([
Promise.all(newActors.map(async actorName => storeActor({ name: actorName }))),
knex('actors_associated').where('release_id', releaseId),
]);
const newlyAssociatedActors = actorEntries
.concat(newActorEntries)
.filter(actorEntry => !associatedActors.some(actor => actorEntry.id === actor.id))
.map(actor => ({
release_id: releaseId,
actor_id: actor.id,
}));
await knex('actors_associated') await knex('actors_associated')
.insert(actorEntries.concat(newActorEntries).map(actor => ({ .insert(newlyAssociatedActors);
release_id: releaseEntry.id,
actor_id: actor.id,
})), '*');
scrapeActors(newActorEntries.map(actor => actor.name));
} }
module.exports = { module.exports = {
associateActors,
fetchActors, fetchActors,
scrapeActors, scrapeActors,
storeActors, scrapeBasicActors,
}; };

11
src/app.js
View File

@ -6,7 +6,7 @@ const initServer = require('./web/server');
const scrapeSites = require('./scrape-sites'); const scrapeSites = require('./scrape-sites');
const scrapeRelease = require('./scrape-release'); const scrapeRelease = require('./scrape-release');
const { scrapeActors } = require('./actors'); const { scrapeActors, scrapeBasicActors } = require('./actors');
async function init() { async function init() {
if (argv.url) { if (argv.url) {
@ -24,13 +24,20 @@ async function init() {
return; return;
} }
if (argv.actors) { if (argv.actors && argv.actors.length > 0) {
await scrapeActors(); await scrapeActors();
knex.destroy(); knex.destroy();
return; return;
} }
if (argv.actors) {
await scrapeBasicActors();
knex.destroy();
return;
}
await initServer(); await initServer();
} }

40
src/media.js
View File

@ -37,13 +37,13 @@ async function createMediaDirectory(release, releaseId) {
} }
} }
async function storePoster(release, releaseEntry) { async function storePoster(release, releaseId) {
if (!release.poster) { if (!release.poster) {
console.warn(`No poster available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`); console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return; return;
} }
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`); console.log(`Storing poster for (${release.site.name}, ${releaseId}) "${release.title}"`);
const res = await bhttp.get(release.poster); const res = await bhttp.get(release.poster);
const thumbnail = await getThumbnail(res.body); const thumbnail = await getThumbnail(res.body);
@ -53,8 +53,8 @@ async function storePoster(release, releaseEntry) {
const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg'; const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
const extension = mime.getExtension(mimetype); const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster.${extension}`); const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster_thumb.${extension}`); const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster_thumb.${extension}`);
const hash = getHash(res.body); const hash = getHash(res.body);
await Promise.all([ await Promise.all([
@ -69,23 +69,23 @@ async function storePoster(release, releaseEntry) {
hash, hash,
source: release.poster, source: release.poster,
domain: 'releases', domain: 'releases',
target_id: releaseEntry.id, target_id: releaseId,
role: 'poster', role: 'poster',
}); });
return; return;
} }
console.warn(`Failed to store poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`); console.warn(`Failed to store poster for (${release.site.name}, ${releaseId}) "${release.title}": ${res.statusCode}`);
} }
async function storePhotos(release, releaseEntry) { async function storePhotos(release, releaseId) {
if (release.photos.length === 0) { if (!release.photos || release.photos.length === 0) {
console.warn(`No photos available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`); console.warn(`No photos available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return; return;
} }
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`); console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
const files = await Promise.map(release.photos, async (photoUrl, index) => { const files = await Promise.map(release.photos, async (photoUrl, index) => {
const { pathname } = new URL(photoUrl); const { pathname } = new URL(photoUrl);
@ -98,8 +98,8 @@ async function storePhotos(release, releaseEntry) {
if (res.statusCode === 200) { if (res.statusCode === 200) {
const extension = mime.getExtension(mimetype); const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}.${extension}`); const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}_thumb.${extension}`); const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body); const hash = getHash(res.body);
await Promise.all([ await Promise.all([
@ -118,7 +118,7 @@ async function storePhotos(release, releaseEntry) {
throw new Error(`Response ${res.statusCode} not OK`); throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) { } catch (error) {
console.warn(`Failed to store photo ${index + 1} for "${release.title}" (${photoUrl}, ${release.url}, ${release.site.name}, ${releaseEntry.id}): ${error}`); console.warn(`Failed to store photo ${index + 1} for "${release.title}" (${photoUrl}, ${release.url}, ${release.site.name}, ${releaseId}): ${error}`);
return null; return null;
} }
@ -136,24 +136,24 @@ async function storePhotos(release, releaseEntry) {
source: file.source, source: file.source,
index, index,
domain: 'releases', domain: 'releases',
target_id: releaseEntry.id, target_id: releaseId,
role: 'photo', role: 'photo',
}))); })));
} }
async function storeTrailer(release, releaseEntry) { async function storeTrailer(release, releaseId) {
if (!release.trailer || !release.trailer.src) { if (!release.trailer || !release.trailer.src) {
console.warn(`No trailer available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`); console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return; return;
} }
console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`); console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`);
const { pathname } = new URL(release.trailer.src); const { pathname } = new URL(release.trailer.src);
const mimetype = release.trailer.type || mime.getType(pathname); const mimetype = release.trailer.type || mime.getType(pathname);
const res = await bhttp.get(release.trailer.src); const res = await bhttp.get(release.trailer.src);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`); const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
await Promise.all([ await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body), fs.writeFile(path.join(config.media.path, filepath), res.body),
@ -162,7 +162,7 @@ async function storeTrailer(release, releaseEntry) {
mime: mimetype, mime: mimetype,
source: release.trailer.src, source: release.trailer.src,
domain: 'releases', domain: 'releases',
target_id: releaseEntry.id, target_id: releaseId,
role: 'trailer', role: 'trailer',
quality: release.trailer.quality || null, quality: release.trailer.quality || null,
}), }),

72
src/releases.js
View File

@ -4,8 +4,8 @@ const Promise = require('bluebird');
const knex = require('./knex'); const knex = require('./knex');
const argv = require('./argv'); const argv = require('./argv');
const whereOr = require('./utils/where-or'); const whereOr = require('./utils/where-or');
const { storeTags } = require('./tags'); const { associateTags } = require('./tags');
const { storeActors } = require('./actors'); const { associateActors } = require('./actors');
const { const {
createMediaDirectory, createMediaDirectory,
storePoster, storePoster,
@ -141,15 +141,6 @@ async function fetchReleases(queryObject = {}, options = {}) {
return curateReleases(releases); return curateReleases(releases);
} }
async function fetchReleasesByEntryIds(entryIds, queryObject = {}, options = {}) {
const releases = await knex('releases')
.modify(commonQuery, options)
.whereIn('entry_id', entryIds)
.andWhere(builder => whereOr(queryObject, 'releases', builder));
return curateReleases(releases);
}
async function fetchSiteReleases(queryObject, options = {}) { async function fetchSiteReleases(queryObject, options = {}) {
const releases = await knex('releases') const releases = await knex('releases')
.modify(commonQuery, options) .modify(commonQuery, options)
@ -192,41 +183,52 @@ async function fetchTagReleases(queryObject, options = {}) {
return curateReleases(releases); return curateReleases(releases);
} }
async function storeReleaseAssets(release, releaseId) {
await createMediaDirectory(release, releaseId);
await Promise.all([
associateActors(release, releaseId),
associateTags(release, releaseId),
storePhotos(release, releaseId),
storePoster(release, releaseId),
storeTrailer(release, releaseId),
]);
}
async function storeRelease(release) { async function storeRelease(release) {
const existingRelease = await knex('releases').where('entry_id', release.entryId).first();
const curatedRelease = curateScrapedRelease(release); const curatedRelease = curateScrapedRelease(release);
const releaseEntries = await knex('releases') if (existingRelease && !argv.redownload) {
return existingRelease.id;
}
if (existingRelease && argv.redownload) {
const [updatedRelease] = await knex('releases')
.where('entry_id', existingRelease.id)
.update({
...existingRelease,
...curatedRelease,
})
.returning('*');
await storeReleaseAssets(release, existingRelease.id);
console.log(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
return updatedRelease || existingRelease;
}
const [releaseEntry] = await knex('releases')
.insert(curatedRelease) .insert(curatedRelease)
.returning('*'); .returning('*');
if (releaseEntries.length) { await storeReleaseAssets(release, releaseEntry.id);
const releaseEntry = releaseEntries[0]; console.log(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
await createMediaDirectory(release, releaseEntry.id);
await Promise.all([
storeActors(release, releaseEntry),
storeTags(release, releaseEntry),
storePhotos(release, releaseEntry),
storePoster(release, releaseEntry),
storeTrailer(release, releaseEntry),
]);
return releaseEntry.id;
}
console.error(`Unable to save scene to database, possible collision: "${release.title}" (${release.site.name})`);
return null; return null;
} }
async function storeReleases(releases) { async function storeReleases(releases) {
const existingReleases = await fetchReleasesByEntryIds(releases.map(release => release.entryId));
console.log(existingReleases);
return Promise.map(releases, async (release) => { return Promise.map(releases, async (release) => {
try { try {
const releaseId = await storeRelease(release); const releaseId = await storeRelease(release);

6
src/scrape-release.js
View File

@ -7,6 +7,7 @@ const scrapers = require('./scrapers/scrapers');
const { storeReleases } = require('./releases'); const { storeReleases } = require('./releases');
const { findSiteByUrl } = require('./sites'); const { findSiteByUrl } = require('./sites');
const { findNetworkByUrl } = require('./networks'); const { findNetworkByUrl } = require('./networks');
const { scrapeBasicActors } = require('./actors');
async function findSite(url, release) { async function findSite(url, release) {
const site = (release && release.site) || await findSiteByUrl(url); const site = (release && release.site) || await findSiteByUrl(url);
@ -48,7 +49,10 @@ async function scrapeRelease(url, release, deep = false) {
if (!deep && argv.save) { if (!deep && argv.save) {
// don't store release when called by site scraper // don't store release when called by site scraper
const releaseId = await storeReleases([scene]); const releaseId = await Promise.all([
storeReleases([scene]),
scrapeBasicActors(),
]);
console.log(`http://${config.web.host}:${config.web.port}/scene/${releaseId}`); console.log(`http://${config.web.host}:${config.web.port}/scene/${releaseId}`);
} }

5
src/scrape-sites.js
View File

@ -9,6 +9,7 @@ const { fetchIncludedSites } = require('./sites');
const scrapers = require('./scrapers/scrapers'); const scrapers = require('./scrapers/scrapers');
const scrapeRelease = require('./scrape-release'); const scrapeRelease = require('./scrape-release');
const { storeReleases } = require('./releases'); const { storeReleases } = require('./releases');
const { scrapeBasicActors } = require('./actors');
function getAfterDate() { function getAfterDate() {
return moment return moment
@ -58,7 +59,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
async function scrapeUpcomingReleases(scraper, site) { async function scrapeUpcomingReleases(scraper, site) {
if (scraper.fetchUpcoming) { if (scraper.fetchUpcoming) {
const upcomingReleases = scraper.fetchUpcoming(site); const upcomingReleases = await scraper.fetchUpcoming(site);
return upcomingReleases.map(release => ({ ...release, upcoming: true })); return upcomingReleases.map(release => ({ ...release, upcoming: true }));
} }
@ -131,6 +132,8 @@ async function scrapeReleases() {
}, { }, {
concurrency: 2, concurrency: 2,
}); });
await scrapeBasicActors();
} }
module.exports = scrapeReleases; module.exports = scrapeReleases;

15
src/scrapers/freeones.js
View File

@ -19,7 +19,9 @@ async function scrapeActorFrontpage(html, url, name) {
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const birthdateString = bio['Date of Birth:']; const birthdateString = bio['Date of Birth:'];
const birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); const birthdate = birthdateString && birthdateString !== 'Unknown (Add)'
? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate()
: null;
const boobsSizeString = bio['Measurements:']; const boobsSizeString = bio['Measurements:'];
const boobsSize = boobsSizeString === '??-??-??' ? null : boobsSizeString; const boobsSize = boobsSizeString === '??-??-??' ? null : boobsSizeString;
@ -74,8 +76,9 @@ async function scrapeActorBio(html, frontpageBio, url, name) {
const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {}); const bio = keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {});
const birthdateString = bio['Date of Birth:']; const birthdateString = bio['Date of Birth:'];
const birthdate = moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate(); const birthdate = birthdateString && birthdateString !== 'Unknown'
const active = bio['Career Status:'].trim() === 'Active'; ? moment.utc(birthdateString.slice(0, birthdateString.indexOf(' (')), 'MMMM D, YYYY').toDate()
: null;
const boobsSizeString = bio['Measurements:']; const boobsSizeString = bio['Measurements:'];
const boobsSize = boobsSizeString === '??-??-??' ? null : boobsSizeString; const boobsSize = boobsSizeString === '??-??-??' ? null : boobsSizeString;
@ -114,7 +117,6 @@ async function scrapeActorBio(html, frontpageBio, url, name) {
eyes, eyes,
piercings, piercings,
tattoos, tattoos,
active,
social, social,
}; };
} }
@ -124,11 +126,16 @@ async function fetchActor(actorName) {
const frontpageUrl = `https://freeones.com/html/v_links/${slug}`; const frontpageUrl = `https://freeones.com/html/v_links/${slug}`;
const resFrontpage = await bhttp.get(frontpageUrl); const resFrontpage = await bhttp.get(frontpageUrl);
if (resFrontpage.statusCode === 200) {
const { url, bio } = await scrapeActorFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName); const { url, bio } = await scrapeActorFrontpage(resFrontpage.body.toString(), frontpageUrl, actorName);
const resBio = await bhttp.get(url); const resBio = await bhttp.get(url);
return scrapeActorBio(resBio.body.toString(), bio, url, actorName); return scrapeActorBio(resBio.body.toString(), bio, url, actorName);
}
return null;
} }
module.exports = { module.exports = {

8
src/tags.js
View File

@ -24,15 +24,15 @@ function curateTags(tags) {
return Promise.all(tags.map(async tag => curateTag(tag))); return Promise.all(tags.map(async tag => curateTag(tag)));
} }
async function storeTags(release, releaseEntry) { async function associateTags(release, releaseId) {
if (!release.tags || release.tags.length === 0) { if (!release.tags || release.tags.length === 0) {
console.warn(`No tags available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`); console.warn(`No tags available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return; return;
} }
await knex('tags_associated').insert(release.tags.map(tagId => ({ await knex('tags_associated').insert(release.tags.map(tagId => ({
tag_id: tagId, tag_id: tagId,
release_id: releaseEntry.id, release_id: releaseId,
}))); })));
} }
@ -74,7 +74,7 @@ async function matchTags(rawTags) {
} }
module.exports = { module.exports = {
storeTags, associateTags,
fetchTags, fetchTags,
matchTags, matchTags,
}; };