DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Refactored database to separate IDs from slugs

This commit is contained in:
ThePendulum 2019-09-08 03:53:09 +02:00
parent 6e7180582b
commit 926c227b85
7 changed files with 3455 additions and 3278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
migrations/20190325001339_releases.js
View File

@ -2,65 +2,67 @@
exports.up = knex => Promise.resolve() exports.up = knex => Promise.resolve()
.then(() => knex.schema.createTable('actors', (table) => { .then(() => knex.schema.createTable('actors', (table) => {
table.increments('id', 8); table.increments('id', 12);
table.string('name') table.string('name')
.unique() .unique()
.notNullable(); .notNullable();
table.string('gender', 18); table.string('gender', 18);
table.integer('alias_for', 8) table.integer('alias_for', 12)
.references('id') .references('id')
.inTable('actors'); .inTable('actors');
table.integer('photos', 3) table.string('slug', 32)
.defaultTo(0); .unique();
})) }))
.then(() => knex.schema.createTable('directors', (table) => { .then(() => knex.schema.createTable('directors', (table) => {
table.increments('id', 8); table.increments('id', 12);
table.string('name'); table.string('name');
table.integer('alias_for', 8) table.integer('alias_for', 12)
.references('id') .references('id')
.inTable('directors'); .inTable('directors');
table.integer('photos', 3) table.string('slug', 32)
.defaultTo(0); .unique();
})) }))
.then(() => knex.schema.createTable('tags_groups', (table) => { .then(() => knex.schema.createTable('tags_groups', (table) => {
table.string('group', 20) table.increments('id', 12);
.primary();
table.string('name', 32); table.string('group', 32);
table.string('slug', 32)
.unique();
})) }))
.then(() => knex.schema.createTable('tags', (table) => { .then(() => knex.schema.createTable('tags', (table) => {
table.string('tag', 32) table.increments('id', 12);
.primary(); table.string('tag');
table.integer('capitalization', 1) table.integer('group_id', 12)
.defaultTo(0); .references('id')
table.string('group_id', 20)
.references('group')
.inTable('tags_groups'); .inTable('tags_groups');
table.string('alias_for', 32) table.integer('alias_for', 12)
.references('tag') .references('id')
.inTable('tags'); .inTable('tags');
table.string('slug', 32)
.unique();
})) }))
.then(() => knex.schema.createTable('networks', (table) => { .then(() => knex.schema.createTable('networks', (table) => {
table.string('id', 32) table.increments('id', 12);
.primary();
table.string('name'); table.string('name');
table.string('url'); table.string('url');
table.text('description'); table.text('description');
table.string('slug', 32)
.unique();
})) }))
.then(() => knex.schema.createTable('sites', (table) => { .then(() => knex.schema.createTable('sites', (table) => {
table.string('id', 32) table.increments('id', 12);
.primary();
table.string('network_id', 32) table.integer('network_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('networks'); .inTable('networks');
@ -69,11 +71,14 @@ exports.up = knex => Promise.resolve()
table.string('url'); table.string('url');
table.text('description'); table.text('description');
table.string('parameters'); table.string('parameters');
table.string('slug', 32)
.unique();
})) }))
.then(() => knex.schema.createTable('releases', (table) => { .then(() => knex.schema.createTable('releases', (table) => {
table.increments('id', 12); table.increments('id', 16);
table.string('site_id', 32) table.integer('site_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('sites'); .inTable('sites');
@ -88,16 +93,13 @@ exports.up = knex => Promise.resolve()
table.date('date'); table.date('date');
table.text('description'); table.text('description');
table.integer('director', 8) table.integer('director', 12)
.references('id') .references('id')
.inTable('directors'); .inTable('directors');
table.integer('duration') table.integer('duration')
.unsigned(); .unsigned();
table.integer('photos', 3)
.defaultTo(0);
table.integer('likes') table.integer('likes')
.unsigned(); .unsigned();
@ -110,10 +112,22 @@ exports.up = knex => Promise.resolve()
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
})) }))
.then(() => knex.schema.createTable('media', (table) => {
table.increments('id', 16);
table.string('file');
table.integer('index');
table.string('mime');
table.enum('domain', ['networks', 'sites', 'releases', 'actors', 'directors']);
table.integer('target_id', 16);
table.enum('role', ['poster', 'logo', 'profile']);
}))
.then(() => knex.schema.createTable('actors_associated', (table) => { .then(() => knex.schema.createTable('actors_associated', (table) => {
table.increments('id', 16); table.increments('id', 16);
table.integer('release_id', 12) table.integer('release_id', 16)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('releases'); .inTable('releases');
@ -126,7 +140,7 @@ exports.up = knex => Promise.resolve()
.then(() => knex.schema.createTable('directors_associated', (table) => { .then(() => knex.schema.createTable('directors_associated', (table) => {
table.increments('id', 16); table.increments('id', 16);
table.integer('release_id', 12) table.integer('release_id', 16)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('releases'); .inTable('releases');
@ -137,16 +151,16 @@ exports.up = knex => Promise.resolve()
.inTable('directors'); .inTable('directors');
})) }))
.then(() => knex.schema.createTable('tags_associated', (table) => { .then(() => knex.schema.createTable('tags_associated', (table) => {
table.string('tag_id', 20) table.integer('tag_id', 12)
.notNullable() .notNullable()
.references('tag') .references('id')
.inTable('tags'); .inTable('tags');
table.string('site_id') table.integer('site_id', 12)
.references('id') .references('id')
.inTable('sites'); .inTable('sites');
table.integer('release_id', 12) table.integer('release_id', 16)
.references('id') .references('id')
.inTable('releases'); .inTable('releases');
})); }));
@ -157,6 +171,7 @@ exports.down = knex => Promise.resolve()
.then(() => knex.schema.dropTable('actors_associated')) .then(() => knex.schema.dropTable('actors_associated'))
.then(() => knex.schema.dropTable('tags')) .then(() => knex.schema.dropTable('tags'))
.then(() => knex.schema.dropTable('tags_groups')) .then(() => knex.schema.dropTable('tags_groups'))
.then(() => knex.schema.dropTable('media'))
.then(() => knex.schema.dropTable('actors')) .then(() => knex.schema.dropTable('actors'))
.then(() => knex.schema.dropTable('releases')) .then(() => knex.schema.dropTable('releases'))
.then(() => knex.schema.dropTable('sites')) .then(() => knex.schema.dropTable('sites'))

32
seeds/00_networks.js
View File

@ -4,96 +4,96 @@
exports.seed = knex => Promise.resolve() exports.seed = knex => Promise.resolve()
.then(() => knex.raw(`${knex('networks').insert([ .then(() => knex.raw(`${knex('networks').insert([
{ {
id: '21sextury', slug: '21sextury',
name: '21Sextury', name: '21Sextury',
url: 'https://www.21sextury.com', url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.', description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
}, },
{ {
id: 'bangbros', slug: 'bangbros',
name: 'Bang Bros', name: 'Bang Bros',
url: 'https://bangbros.com', url: 'https://bangbros.com',
description: 'Here at Bang Bros, we only film the best highest quality porn with the sexiest Amateur girls and the top pornstars. Updated daily on Bangbros.com.', description: 'Here at Bang Bros, we only film the best highest quality porn with the sexiest Amateur girls and the top pornstars. Updated daily on Bangbros.com.',
}, },
{ {
id: 'blowpass', slug: 'blowpass',
name: 'Blowpass', name: 'Blowpass',
url: 'https://www.blowpass.com', url: 'https://www.blowpass.com',
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!', description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
}, },
{ {
id: 'brazzers', slug: 'brazzers',
name: 'Brazzers', name: 'Brazzers',
url: 'https://www.brazzers.com', url: 'https://www.brazzers.com',
description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.', description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
}, },
{ {
id: 'ddfnetwork', slug: 'ddfnetwork',
name: 'DDF Network', name: 'DDF Network',
url: 'https://ddfnetwork.com', url: 'https://ddfnetwork.com',
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.', description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
}, },
{ {
id: 'julesjordan', slug: 'julesjordan',
name: 'Jules Jordan', name: 'Jules Jordan',
url: 'https://www.julesjordan.com', url: 'https://www.julesjordan.com',
}, },
{ {
id: 'kink', slug: 'kink',
name: 'Kink', name: 'Kink',
url: 'https://www.kink.com', url: 'https://www.kink.com',
description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.', description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
}, },
{ {
id: 'legalporno', slug: 'legalporno',
name: 'LegalPorno', name: 'LegalPorno',
url: 'https://www.legalporno.com', url: 'https://www.legalporno.com',
description: 'The Best HD Porn For You!', description: 'The Best HD Porn For You!',
}, },
{ {
id: 'mofos', slug: 'mofos',
name: 'MOFOS', name: 'MOFOS',
url: 'https://www.mofos.com', url: 'https://www.mofos.com',
description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.', description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
}, },
{ {
id: 'naughtyamerica', slug: 'naughtyamerica',
name: 'Naughty America', name: 'Naughty America',
url: 'https://www.naughtyamerica.com', url: 'https://www.naughtyamerica.com',
description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!', description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!',
}, },
{ {
id: 'pervcity', slug: 'pervcity',
name: 'Perv City', name: 'Perv City',
url: 'https://www.pervcity.com', url: 'https://www.pervcity.com',
description: '', description: '',
}, },
{ {
id: 'pornpros', slug: 'pornpros',
name: 'Porn Pros', name: 'Porn Pros',
url: 'https://pornpros.com', url: 'https://pornpros.com',
description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.', description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.',
}, },
{ {
id: 'private', slug: 'private',
name: 'Private', name: 'Private',
url: 'https://www.private.com', url: 'https://www.private.com',
description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...', description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
}, },
{ {
id: 'realitykings', slug: 'realitykings',
name: 'Reality Kings', name: 'Reality Kings',
url: 'https://www.realitykings.com', url: 'https://www.realitykings.com',
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com', description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
}, },
{ {
id: 'vixen', slug: 'vixen',
name: 'Vixen', name: 'Vixen',
url: 'https://www.vixen.com', url: 'https://www.vixen.com',
description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.', description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
}, },
{ {
id: 'xempire', slug: 'xempire',
name: 'XEmpire', name: 'XEmpire',
url: 'https://www.xempire.com', url: 'https://www.xempire.com',
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!', description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',

1170
seeds/01_sites.js
View File

File diff suppressed because it is too large Load Diff

652
seeds/02_tags.js
View File

File diff suppressed because it is too large Load Diff

4
src/argv.js
View File

@ -13,7 +13,7 @@ const { argv } = yargs
.option('deep', { .option('deep', {
describe: 'Fetch details for all releases', describe: 'Fetch details for all releases',
type: 'boolean', type: 'boolean',
default: false, default: true,
}) })
.option('networks', { .option('networks', {
describe: 'Networks to include (overrides config)', describe: 'Networks to include (overrides config)',
@ -33,7 +33,7 @@ const { argv } = yargs
.option('save', { .option('save', {
describe: 'Save fetched releases to database', describe: 'Save fetched releases to database',
type: 'boolean', type: 'boolean',
default: false, default: true,
}) })
.option('render', { .option('render', {
describe: 'Fetch data without rendering interface', describe: 'Fetch data without rendering interface',

32
src/fetch-releases.js
View File

@ -77,6 +77,23 @@ async function findDuplicateReleases(latestReleases, _siteId) {
.orWhereIn('entry_id', latestReleasesEntryIds); .orWhereIn('entry_id', latestReleasesEntryIds);
} }
async function storePhotos(release, releaseEntry) {
const photoPath = path.join(config.photoPath, release.site.id, releaseEntry.rows[0].id.toString());
await fs.mkdir(photoPath, { recursive: true });
console.log(`Storing photos for (${release.site.name}, ${release.id}) "${release.title}"`);
await Promise.map(release.photos, async (photoUrl, index) => {
const res = await bhttp.get(photoUrl);
await fs.writeFile(path.join(photoPath, `${index + 1}.jpg`), res.body);
return photoUrl;
}, {
concurrency: 2,
});
}
async function storeReleases(releases = []) { async function storeReleases(releases = []) {
return Promise.map(releases, async (release) => { return Promise.map(releases, async (release) => {
const curatedRelease = { const curatedRelease = {
@ -121,20 +138,7 @@ async function storeReleases(releases = []) {
} }
if (release.photos && release.photos.length > 0) { if (release.photos && release.photos.length > 0) {
const photoPath = path.join(config.photoPath, release.site.id, releaseEntry.rows[0].id.toString()); await storePhotos(release, releaseEntry);
await fs.mkdir(photoPath, { recursive: true });
console.log(`Storing photos for (${release.site.name}, ${release.id}) "${release.title}"`);
await Promise.map(release.photos, async (photoUrl, index) => {
const res = await bhttp.get(photoUrl);
await fs.writeFile(path.join(photoPath, `${index + 1}.jpg`), res.body);
return photoUrl;
}, {
concurrency: 2,
});
} }
}, { }, {
concurrency: 2, concurrency: 2,

2
src/scrapers/kink.js
View File

@ -52,7 +52,7 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const title = $('h1.shoot-title span.favorite-button').attr('data-title'); const title = $('h1.shoot-title span.favorite-button').attr('data-title');
const actorsRaw = $('.shoot-info p.starring'); const actorsRaw = $('.shoot-info p.starring');
const photos = $('.gallery .thumb img').map((photoIndex, photoElement) => `https://cdnp.kink.com${$(photoElement).attr('data-image-file')}`).toArray(); const photos = $('.gallery .thumb img').map((photoIndex, photoElement) => $(photoElement).attr('data-image-file')).toArray();
const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url'); const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url');
const trailerPoster = $('.player video#kink-player').attr('poster'); const trailerPoster = $('.player video#kink-player').attr('poster');