Refactored database to separate IDs from slugs
This commit is contained in:
parent
6e7180582b
commit
926c227b85
|
|
@ -2,65 +2,67 @@
|
|||
|
||||
exports.up = knex => Promise.resolve()
|
||||
.then(() => knex.schema.createTable('actors', (table) => {
|
||||
table.increments('id', 8);
|
||||
table.increments('id', 12);
|
||||
|
||||
table.string('name')
|
||||
.unique()
|
||||
.notNullable();
|
||||
table.string('gender', 18);
|
||||
|
||||
table.integer('alias_for', 8)
|
||||
table.integer('alias_for', 12)
|
||||
.references('id')
|
||||
.inTable('actors');
|
||||
|
||||
table.integer('photos', 3)
|
||||
.defaultTo(0);
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('directors', (table) => {
|
||||
table.increments('id', 8);
|
||||
table.increments('id', 12);
|
||||
|
||||
table.string('name');
|
||||
table.integer('alias_for', 8)
|
||||
table.integer('alias_for', 12)
|
||||
.references('id')
|
||||
.inTable('directors');
|
||||
|
||||
table.integer('photos', 3)
|
||||
.defaultTo(0);
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('tags_groups', (table) => {
|
||||
table.string('group', 20)
|
||||
.primary();
|
||||
table.increments('id', 12);
|
||||
|
||||
table.string('name', 32);
|
||||
table.string('group', 32);
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('tags', (table) => {
|
||||
table.string('tag', 32)
|
||||
.primary();
|
||||
table.increments('id', 12);
|
||||
table.string('tag');
|
||||
|
||||
table.integer('capitalization', 1)
|
||||
.defaultTo(0);
|
||||
|
||||
table.string('group_id', 20)
|
||||
.references('group')
|
||||
table.integer('group_id', 12)
|
||||
.references('id')
|
||||
.inTable('tags_groups');
|
||||
|
||||
table.string('alias_for', 32)
|
||||
.references('tag')
|
||||
table.integer('alias_for', 12)
|
||||
.references('id')
|
||||
.inTable('tags');
|
||||
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('networks', (table) => {
|
||||
table.string('id', 32)
|
||||
.primary();
|
||||
table.increments('id', 12);
|
||||
|
||||
table.string('name');
|
||||
table.string('url');
|
||||
table.text('description');
|
||||
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('sites', (table) => {
|
||||
table.string('id', 32)
|
||||
.primary();
|
||||
table.increments('id', 12);
|
||||
|
||||
table.string('network_id', 32)
|
||||
table.integer('network_id', 12)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('networks');
|
||||
|
|
@ -69,11 +71,14 @@ exports.up = knex => Promise.resolve()
|
|||
table.string('url');
|
||||
table.text('description');
|
||||
table.string('parameters');
|
||||
|
||||
table.string('slug', 32)
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.schema.createTable('releases', (table) => {
|
||||
table.increments('id', 12);
|
||||
table.increments('id', 16);
|
||||
|
||||
table.string('site_id', 32)
|
||||
table.integer('site_id', 12)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('sites');
|
||||
|
|
@ -88,16 +93,13 @@ exports.up = knex => Promise.resolve()
|
|||
table.date('date');
|
||||
table.text('description');
|
||||
|
||||
table.integer('director', 8)
|
||||
table.integer('director', 12)
|
||||
.references('id')
|
||||
.inTable('directors');
|
||||
|
||||
table.integer('duration')
|
||||
.unsigned();
|
||||
|
||||
table.integer('photos', 3)
|
||||
.defaultTo(0);
|
||||
|
||||
table.integer('likes')
|
||||
.unsigned();
|
||||
|
||||
|
|
@ -110,10 +112,22 @@ exports.up = knex => Promise.resolve()
|
|||
table.datetime('created_at')
|
||||
.defaultTo(knex.fn.now());
|
||||
}))
|
||||
.then(() => knex.schema.createTable('media', (table) => {
|
||||
table.increments('id', 16);
|
||||
|
||||
table.string('file');
|
||||
table.integer('index');
|
||||
table.string('mime');
|
||||
|
||||
table.enum('domain', ['networks', 'sites', 'releases', 'actors', 'directors']);
|
||||
table.integer('target_id', 16);
|
||||
|
||||
table.enum('role', ['poster', 'logo', 'profile']);
|
||||
}))
|
||||
.then(() => knex.schema.createTable('actors_associated', (table) => {
|
||||
table.increments('id', 16);
|
||||
|
||||
table.integer('release_id', 12)
|
||||
table.integer('release_id', 16)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('releases');
|
||||
|
|
@ -126,7 +140,7 @@ exports.up = knex => Promise.resolve()
|
|||
.then(() => knex.schema.createTable('directors_associated', (table) => {
|
||||
table.increments('id', 16);
|
||||
|
||||
table.integer('release_id', 12)
|
||||
table.integer('release_id', 16)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('releases');
|
||||
|
|
@ -137,16 +151,16 @@ exports.up = knex => Promise.resolve()
|
|||
.inTable('directors');
|
||||
}))
|
||||
.then(() => knex.schema.createTable('tags_associated', (table) => {
|
||||
table.string('tag_id', 20)
|
||||
table.integer('tag_id', 12)
|
||||
.notNullable()
|
||||
.references('tag')
|
||||
.references('id')
|
||||
.inTable('tags');
|
||||
|
||||
table.string('site_id')
|
||||
table.integer('site_id', 12)
|
||||
.references('id')
|
||||
.inTable('sites');
|
||||
|
||||
table.integer('release_id', 12)
|
||||
table.integer('release_id', 16)
|
||||
.references('id')
|
||||
.inTable('releases');
|
||||
}));
|
||||
|
|
@ -157,6 +171,7 @@ exports.down = knex => Promise.resolve()
|
|||
.then(() => knex.schema.dropTable('actors_associated'))
|
||||
.then(() => knex.schema.dropTable('tags'))
|
||||
.then(() => knex.schema.dropTable('tags_groups'))
|
||||
.then(() => knex.schema.dropTable('media'))
|
||||
.then(() => knex.schema.dropTable('actors'))
|
||||
.then(() => knex.schema.dropTable('releases'))
|
||||
.then(() => knex.schema.dropTable('sites'))
|
||||
|
|
|
|||
|
|
@ -4,96 +4,96 @@
|
|||
exports.seed = knex => Promise.resolve()
|
||||
.then(() => knex.raw(`${knex('networks').insert([
|
||||
{
|
||||
id: '21sextury',
|
||||
slug: '21sextury',
|
||||
name: '21Sextury',
|
||||
url: 'https://www.21sextury.com',
|
||||
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
|
||||
},
|
||||
{
|
||||
id: 'bangbros',
|
||||
slug: 'bangbros',
|
||||
name: 'Bang Bros',
|
||||
url: 'https://bangbros.com',
|
||||
description: 'Here at Bang Bros, we only film the best highest quality porn with the sexiest Amateur girls and the top pornstars. Updated daily on Bangbros.com.',
|
||||
},
|
||||
{
|
||||
id: 'blowpass',
|
||||
slug: 'blowpass',
|
||||
name: 'Blowpass',
|
||||
url: 'https://www.blowpass.com',
|
||||
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
|
||||
},
|
||||
{
|
||||
id: 'brazzers',
|
||||
slug: 'brazzers',
|
||||
name: 'Brazzers',
|
||||
url: 'https://www.brazzers.com',
|
||||
description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
|
||||
},
|
||||
{
|
||||
id: 'ddfnetwork',
|
||||
slug: 'ddfnetwork',
|
||||
name: 'DDF Network',
|
||||
url: 'https://ddfnetwork.com',
|
||||
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
|
||||
},
|
||||
{
|
||||
id: 'julesjordan',
|
||||
slug: 'julesjordan',
|
||||
name: 'Jules Jordan',
|
||||
url: 'https://www.julesjordan.com',
|
||||
},
|
||||
{
|
||||
id: 'kink',
|
||||
slug: 'kink',
|
||||
name: 'Kink',
|
||||
url: 'https://www.kink.com',
|
||||
description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
|
||||
},
|
||||
{
|
||||
id: 'legalporno',
|
||||
slug: 'legalporno',
|
||||
name: 'LegalPorno',
|
||||
url: 'https://www.legalporno.com',
|
||||
description: 'The Best HD Porn For You!',
|
||||
},
|
||||
{
|
||||
id: 'mofos',
|
||||
slug: 'mofos',
|
||||
name: 'MOFOS',
|
||||
url: 'https://www.mofos.com',
|
||||
description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
|
||||
},
|
||||
{
|
||||
id: 'naughtyamerica',
|
||||
slug: 'naughtyamerica',
|
||||
name: 'Naughty America',
|
||||
url: 'https://www.naughtyamerica.com',
|
||||
description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!',
|
||||
},
|
||||
{
|
||||
id: 'pervcity',
|
||||
slug: 'pervcity',
|
||||
name: 'Perv City',
|
||||
url: 'https://www.pervcity.com',
|
||||
description: '',
|
||||
},
|
||||
{
|
||||
id: 'pornpros',
|
||||
slug: 'pornpros',
|
||||
name: 'Porn Pros',
|
||||
url: 'https://pornpros.com',
|
||||
description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.',
|
||||
},
|
||||
{
|
||||
id: 'private',
|
||||
slug: 'private',
|
||||
name: 'Private',
|
||||
url: 'https://www.private.com',
|
||||
description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
|
||||
},
|
||||
{
|
||||
id: 'realitykings',
|
||||
slug: 'realitykings',
|
||||
name: 'Reality Kings',
|
||||
url: 'https://www.realitykings.com',
|
||||
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
|
||||
},
|
||||
{
|
||||
id: 'vixen',
|
||||
slug: 'vixen',
|
||||
name: 'Vixen',
|
||||
url: 'https://www.vixen.com',
|
||||
description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
|
||||
},
|
||||
{
|
||||
id: 'xempire',
|
||||
slug: 'xempire',
|
||||
name: 'XEmpire',
|
||||
url: 'https://www.xempire.com',
|
||||
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
|
||||
|
|
|
|||
4068
seeds/01_sites.js
4068
seeds/01_sites.js
File diff suppressed because it is too large
Load Diff
2506
seeds/02_tags.js
2506
seeds/02_tags.js
File diff suppressed because it is too large
Load Diff
|
|
@ -13,7 +13,7 @@ const { argv } = yargs
|
|||
.option('deep', {
|
||||
describe: 'Fetch details for all releases',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
default: true,
|
||||
})
|
||||
.option('networks', {
|
||||
describe: 'Networks to include (overrides config)',
|
||||
|
|
@ -33,7 +33,7 @@ const { argv } = yargs
|
|||
.option('save', {
|
||||
describe: 'Save fetched releases to database',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
default: true,
|
||||
})
|
||||
.option('render', {
|
||||
describe: 'Fetch data without rendering interface',
|
||||
|
|
|
|||
|
|
@ -77,6 +77,23 @@ async function findDuplicateReleases(latestReleases, _siteId) {
|
|||
.orWhereIn('entry_id', latestReleasesEntryIds);
|
||||
}
|
||||
|
||||
async function storePhotos(release, releaseEntry) {
|
||||
const photoPath = path.join(config.photoPath, release.site.id, releaseEntry.rows[0].id.toString());
|
||||
|
||||
await fs.mkdir(photoPath, { recursive: true });
|
||||
|
||||
console.log(`Storing photos for (${release.site.name}, ${release.id}) "${release.title}"`);
|
||||
|
||||
await Promise.map(release.photos, async (photoUrl, index) => {
|
||||
const res = await bhttp.get(photoUrl);
|
||||
await fs.writeFile(path.join(photoPath, `${index + 1}.jpg`), res.body);
|
||||
|
||||
return photoUrl;
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
}
|
||||
|
||||
async function storeReleases(releases = []) {
|
||||
return Promise.map(releases, async (release) => {
|
||||
const curatedRelease = {
|
||||
|
|
@ -121,20 +138,7 @@ async function storeReleases(releases = []) {
|
|||
}
|
||||
|
||||
if (release.photos && release.photos.length > 0) {
|
||||
const photoPath = path.join(config.photoPath, release.site.id, releaseEntry.rows[0].id.toString());
|
||||
|
||||
await fs.mkdir(photoPath, { recursive: true });
|
||||
|
||||
console.log(`Storing photos for (${release.site.name}, ${release.id}) "${release.title}"`);
|
||||
|
||||
await Promise.map(release.photos, async (photoUrl, index) => {
|
||||
const res = await bhttp.get(photoUrl);
|
||||
await fs.writeFile(path.join(photoPath, `${index + 1}.jpg`), res.body);
|
||||
|
||||
return photoUrl;
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
await storePhotos(release, releaseEntry);
|
||||
}
|
||||
}, {
|
||||
concurrency: 2,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
|
|||
const title = $('h1.shoot-title span.favorite-button').attr('data-title');
|
||||
const actorsRaw = $('.shoot-info p.starring');
|
||||
|
||||
const photos = $('.gallery .thumb img').map((photoIndex, photoElement) => `https://cdnp.kink.com${$(photoElement).attr('data-image-file')}`).toArray();
|
||||
const photos = $('.gallery .thumb img').map((photoIndex, photoElement) => $(photoElement).attr('data-image-file')).toArray();
|
||||
const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url');
|
||||
const trailerPoster = $('.player video#kink-player').attr('poster');
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue