Adding networks and sites as entities,

2020-06-04 01:03:02 +02:00 · 2020-06-04 01:03:02 +02:00 · 09d849eb9d
parent 8abcc7194a
commit 09d849eb9d
9 changed files with 283 additions and 221 deletions
--- a/assets/components/tile/tag.vue
+++ b/assets/components/tile/tag.vue
@ -1,6 +1,6 @@
 <template>
 	<router-link
-		:to="{ name: 'tag', params: { tagSlug: tag.slug } }"
+		:to="{ name: 'tag', params: { tagSlug: tag.slug, tags: 'all-tags', range: 'latest' } }"
 		:title="tag.name"
 		class="tile"
 	>
--- a/config/default.js
+++ b/config/default.js
@ -14,54 +14,49 @@ module.exports = {
 	// include: [],
 	// exclude: [],
 	exclude: [
-		['21sextreme', [
-			// no longer updated
-			'mightymistress',
-			'dominatedgirls',
-			'homepornreality',
-			'peeandblow',
-			'cummingmatures',
-			'mandyiskinky',
-			'speculumplays',
-			'creampiereality',
-		]],
-		['aziani', [
-			'amberathome',
-			'marycarey',
-			'racqueldevonshire',
-		]],
+		// 21sextreme, no longer updated
+		'mightymistress',
+		'dominatedgirls',
+		'homepornreality',
+		'peeandblow',
+		'cummingmatures',
+		'mandyiskinky',
+		'speculumplays',
+		'creampiereality',
+		// aziani
+		'amberathome',
+		'marycarey',
+		'racqueldevonshire',
+		// boobpedia
 		'boobpedia',
-		['blowpass', ['sunlustxxx']],
-		['ddfnetwork', [
-			'fuckinhd',
-			'bustylover',
-		]],
-		['famedigital', [
-			'daringsex',
-			'lowartfilms',
-		]],
+		// blowpass
+		'sunlustxxx',
+		// ddfnetwork
+		'fuckinhd',
+		'bustylover',
+		// famedigital
+		'daringsex',
+		'lowartfilms',
+		// freeones
 		'freeones',
-		['pornpros', [
-			'milfhumiliation',
-			'humiliated',
-			'flexiblepositions',
-			'publicviolations',
-			'amateurviolations',
-			'squirtdisgrace',
-			'cumdisgrace',
-			'webcamhackers',
-			'collegeteens',
-		]],
-		['score', [
-			'bigboobbundle',
-			'milfbundle',
-			'pornmegaload',
-			'scorelandtv',
-			'scoretv',
-		]],
-		['mindgeek', [
-			'pornhub',
-		]],
+		// pornpros
+		'milfhumiliation',
+		'humiliated',
+		'flexiblepositions',
+		'publicviolations',
+		'amateurviolations',
+		'squirtdisgrace',
+		'cumdisgrace',
+		'webcamhackers',
+		'collegeteens',
+		// score
+		'bigboobbundle',
+		'milfbundle',
+		'pornmegaload',
+		'scorelandtv',
+		'scoretv',
+		// mindgeek
+		'pornhub',
 	],
 	profiles: [
 		[
--- a/migrations/20190325001339_releases.js
+++ b/migrations/20190325001339_releases.js
@ -140,6 +140,59 @@ exports.up = knex => Promise.resolve()

 		table.unique(['tag_id', 'media_id']);
 	}))
+	.then(() => knex.schema.createTable('entities_types', (table) => {
+		table.increments('id', 4);
+		table.text('type');
+	}))
+	.then(() => knex('entities_types').insert([
+		{ type: 'network' },
+		{ type: 'channel' },
+		{ type: 'studio' },
+	]))
+	.then(() => knex.schema.createTable('entities', (table) => {
+		table.increments('id', 12);
+
+		table.integer('parent_id', 12)
+			.references('id')
+			.inTable('entities');
+
+		table.integer('type', 4)
+			.references('id')
+			.inTable('entities_types')
+			.defaultTo(2);
+
+		table.text('name');
+		table.text('slug', 32)
+			.unique();
+		table.text('alias');
+
+		table.text('url');
+		table.text('description');
+		table.json('parameters');
+
+		table.boolean('active');
+		table.integer('priority', 3)
+			.defaultTo(0);
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	}))
+	.then(() => knex.schema.createTable('entities_tags', (table) => {
+		table.integer('tag_id', 12)
+			.notNullable()
+			.references('id')
+			.inTable('tags');
+
+		table.integer('entity_id', 12)
+			.notNullable()
+			.references('id')
+			.inTable('entities');
+
+		table.boolean('inherit')
+			.defaultTo(false);
+
+		table.unique(['tag_id', 'entity_id']);
+	}))
 	.then(() => knex.schema.createTable('networks', (table) => {
 		table.increments('id', 12);

@ -897,6 +950,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS actors_piercings CASCADE;
 		DROP TABLE IF EXISTS body CASCADE;

+		DROP TABLE IF EXISTS entities_tags CASCADE;
 		DROP TABLE IF EXISTS sites_tags CASCADE;
 		DROP TABLE IF EXISTS sites_social CASCADE;
 		DROP TABLE IF EXISTS networks_social CASCADE;
@ -915,6 +969,9 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS countries CASCADE;
 		DROP TABLE IF EXISTS networks CASCADE;

+		DROP TABLE IF EXISTS entities_types CASCADE;
+		DROP TABLE IF EXISTS entities CASCADE;
+
 		DROP FUNCTION IF EXISTS search_sites;
 		DROP FUNCTION IF EXISTS search_actors;
 		DROP FUNCTION IF EXISTS get_random_sfw_media_id;
--- a/seeds/01_networks.js
+++ b/seeds/01_networks.js
@ -430,7 +430,7 @@ const networks = [

 exports.seed = knex => Promise.resolve()
 	.then(async () => {
-		const { inserted, updated } = await upsert('networks', parentNetworks, 'slug', knex);
+		const { inserted, updated } = await upsert('entities', parentNetworks, 'slug', knex);
 		const parentNetworksBySlug = [].concat(inserted, updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});

 		const networksWithParent = networks.map(network => ({
@ -443,5 +443,5 @@ exports.seed = knex => Promise.resolve()
 			parent_id: parentNetworksBySlug[network.parent] || null,
 		}));

-		return upsert('networks', networksWithParent, 'slug', knex);
+		return upsert('entities', networksWithParent, 'slug', knex);
 	});
--- a/seeds/02_sites.js
+++ b/seeds/02_sites.js
@ -1158,7 +1158,7 @@ const sites = [
 		url: 'https://www.sunlustxxx.com',
 		description: '',
 		network: 'blowpass',
-		show: true, // site offline, use only for indexing old scenes
+		active: true, // site offline, use only for indexing old scenes
 	},
 	// BOOBPEDIA
 	{
@ -1877,7 +1877,7 @@ const sites = [
 		description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
 		network: 'famedigital',
 		parameters: { api: true },
-		show: false, // no data sources
+		active: false, // no data sources
 	},
 	{
 		slug: 'peternorth',
@ -4157,6 +4157,7 @@ const sites = [
 	{
 		name: 'Teen BFF',
 		slug: 'teenbff',
+		alias: ['tbff'],
 		url: 'https://pornpros.com/site/teenbff',
 		tags: ['mff'],
 		network: 'pornpros',
@ -4921,7 +4922,7 @@ const sites = [
 		slug: 'bigboobbundle',
 		url: 'https://www.bigboobbundle.com',
 		network: 'score',
-		show: false, // all content appears to be on subsites
+		active: false, // all content appears to be on subsites
 	},
 	{
 		name: 'Big Boobs POV',
@ -5213,7 +5214,7 @@ const sites = [
 		slug: 'milfbundle',
 		url: 'https://www.milfbundle.com',
 		network: 'score',
-		show: false,
+		active: false,
 	},
 	{
 		name: 'Teaming Cock',
@ -5280,7 +5281,7 @@ const sites = [
 		slug: 'pornmegaload',
 		url: 'https://www.pornmegaload.com',
 		network: 'score',
-		show: false,
+		active: false,
 	},
 	{
 		name: 'SaRennas World',
@ -5318,7 +5319,7 @@ const sites = [
 		url: 'https://www.scorepass.com/scorelandtv',
 		network: 'score',
 		priority: 1,
-		show: false, // appears to be streaming service for other sites
+		active: false, // appears to be streaming service for other sites
 	},
 	{
 		name: 'ScoreTV',
@ -5326,7 +5327,7 @@ const sites = [
 		url: 'https://www.scoretv.tv',
 		network: 'score',
 		priority: 1,
-		show: false, // similar to or same as Scoreland TV
+		active: false, // similar to or same as Scoreland TV
 	},
 	{
 		name: 'Score Videos',
@ -6177,7 +6178,7 @@ const sites = [
 /* eslint-disable max-len */
 exports.seed = knex => Promise.resolve()
 	.then(async () => {
-		const networks = await knex('networks').select('*');
+		const networks = await knex('entities').select('*');
 		const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});

 		const tags = await knex('tags').select('*').where('alias_for', null);
@ -6190,24 +6191,24 @@ exports.seed = knex => Promise.resolve()
 			description: site.description,
 			url: site.url,
 			parameters: site.parameters,
-			network_id: networksMap[site.network],
+			parent_id: networksMap[site.network],
 			priority: site.priority,
-			show: site.show,
+			active: site.show,
 		}));

-		const { inserted, updated } = await upsert('sites', sitesWithNetworks, 'slug', knex);
+		const { inserted, updated } = await upsert('entities', sitesWithNetworks, 'slug', knex);
 		const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});

 		const tagAssociations = sites.map(site => (site.tags
 			? site.tags.map(tagSlug => ({
-				site_id: sitesMap[site.slug],
+				entity_id: sitesMap[site.slug],
 				tag_id: tagsMap[tagSlug],
 				inherit: true,
 			}))
 			: []
 		)).flat();

-		return upsert('sites_tags', tagAssociations, ['site_id', 'tag_id'], knex);
+		return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
 	});

 /*
--- a/seeds/03_studios.js
+++ b/seeds/03_studios.js
@ -1,162 +1,170 @@
 const upsert = require('../src/utils/upsert');

 function getStudios(networksMap) {
-    return [
-        // LegalPorno
-        {
-            slug: 'gonzocom',
-            name: 'Gonzo.com',
-            url: 'https://www.legalporno.com/studios/gonzo_com',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'giorgiograndi',
-            name: 'Giorgio Grandi',
-            url: 'https://www.legalporno.com/studios/giorgio-grandi',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'hardpornworld',
-            name: 'Hard Porn World',
-            url: 'https://www.legalporno.com/studios/hard-porn-world',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'interracialvision',
-            name: 'Interracial Vision',
-            url: 'https://www.legalporno.com/studios/interracial-vision',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'giorgioslab',
-            name: 'Giorgio\'s Lab',
-            url: 'https://www.legalporno.com/studios/giorgio--s-lab',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'americananal',
-            name: 'American Anal',
-            url: 'https://www.legalporno.com/studios/american-anal',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'assablanca',
-            name: 'Assablanca',
-            url: 'https://www.legalporno.com/studios/assablanca',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'focus',
-            name: 'Focus',
-            url: 'https://www.legalporno.com/studios/focus',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'analforever',
-            name: 'Anal Forever',
-            url: 'https://www.legalporno.com/studios/anal-forever',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'gonzoinbrazil',
-            name: 'Gonzo in Brazil',
-            url: 'https://www.legalporno.com/studios/gonzo-in-brazil',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'mranal',
-            name: 'Mr Anal',
-            url: 'https://www.legalporno.com/studios/mr-anal',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'tarrawhite',
-            name: 'Tarra White',
-            url: 'https://www.legalporno.com/studios/tarra-white',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'sineplexsos',
-            name: 'Sineplex SOS',
-            url: 'https://www.legalporno.com/studios/sineplex-sos',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'fmodels',
-            name: 'F Models',
-            url: 'https://www.legalporno.com/studios/f-models',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'sineplexcz',
-            name: 'Sineplex CZ',
-            url: 'https://www.legalporno.com/studios/sineplex-cz',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'gg',
-            name: 'GG',
-            url: 'https://www.legalporno.com/studios/gg',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'firstgape',
-            name: 'First Gape',
-            url: 'https://www.legalporno.com/studios/first-gape',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'omargalantiproductions',
-            name: 'Omar Galanti Productions',
-            url: 'https://www.legalporno.com/studios/omar-galanti-productions',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'norestfortheass',
-            name: 'No Rest For The Ass',
-            url: 'https://www.legalporno.com/studios/no-rest-for-the-ass',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'hairygonzo',
-            name: 'Hairy Gonzo',
-            url: 'https://www.legalporno.com/studios/hairy-gonzo',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'sineplexclassic',
-            name: 'Sineplex Classic',
-            url: 'https://www.legalporno.com/studios/sineplex-classic',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'sinemale',
-            name: 'Sinemale',
-            url: 'https://www.legalporno.com/studios/sinemale',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'outsidethestudio',
-            name: 'Outside The Studio',
-            url: 'https://www.legalporno.com/studios/outside-the-studio',
-            network_id: networksMap.legalporno,
-        },
-        {
-            slug: 'kinkysex',
-            name: 'Kinky Sex',
-            url: 'https://www.legalporno.com/studios/kinky-sex',
-            network_id: networksMap.legalporno,
-        },
-    ];
+	return [
+		// LegalPorno
+		{
+			slug: 'gonzocom',
+			name: 'Gonzo.com',
+			url: 'https://www.legalporno.com/studios/gonzo_com',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'giorgiograndi',
+			name: 'Giorgio Grandi',
+			url: 'https://www.legalporno.com/studios/giorgio-grandi',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'hardpornworld',
+			name: 'Hard Porn World',
+			url: 'https://www.legalporno.com/studios/hard-porn-world',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'interracialvision',
+			name: 'Interracial Vision',
+			url: 'https://www.legalporno.com/studios/interracial-vision',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'giorgioslab',
+			name: 'Giorgio\'s Lab',
+			url: 'https://www.legalporno.com/studios/giorgio--s-lab',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'americananal',
+			name: 'American Anal',
+			url: 'https://www.legalporno.com/studios/american-anal',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'assablanca',
+			name: 'Assablanca',
+			url: 'https://www.legalporno.com/studios/assablanca',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'focus',
+			name: 'Focus',
+			url: 'https://www.legalporno.com/studios/focus',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'analforever',
+			name: 'Anal Forever',
+			url: 'https://www.legalporno.com/studios/anal-forever',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'gonzoinbrazil',
+			name: 'Gonzo in Brazil',
+			url: 'https://www.legalporno.com/studios/gonzo-in-brazil',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'mranal',
+			name: 'Mr Anal',
+			url: 'https://www.legalporno.com/studios/mr-anal',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'tarrawhite',
+			name: 'Tarra White',
+			url: 'https://www.legalporno.com/studios/tarra-white',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'sineplexsos',
+			name: 'Sineplex SOS',
+			url: 'https://www.legalporno.com/studios/sineplex-sos',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'fmodels',
+			name: 'F Models',
+			url: 'https://www.legalporno.com/studios/f-models',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'sineplexcz',
+			name: 'Sineplex CZ',
+			url: 'https://www.legalporno.com/studios/sineplex-cz',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'gg',
+			name: 'GG',
+			url: 'https://www.legalporno.com/studios/gg',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'firstgape',
+			name: 'First Gape',
+			url: 'https://www.legalporno.com/studios/first-gape',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'omargalantiproductions',
+			name: 'Omar Galanti Productions',
+			url: 'https://www.legalporno.com/studios/omar-galanti-productions',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'norestfortheass',
+			name: 'No Rest For The Ass',
+			url: 'https://www.legalporno.com/studios/no-rest-for-the-ass',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'hairygonzo',
+			name: 'Hairy Gonzo',
+			url: 'https://www.legalporno.com/studios/hairy-gonzo',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'sineplexclassic',
+			name: 'Sineplex Classic',
+			url: 'https://www.legalporno.com/studios/sineplex-classic',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'sinemale',
+			name: 'Sinemale',
+			url: 'https://www.legalporno.com/studios/sinemale',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'outsidethestudio',
+			name: 'Outside The Studio',
+			url: 'https://www.legalporno.com/studios/outside-the-studio',
+			parent_id: networksMap.legalporno,
+		},
+		{
+			slug: 'kinkysex',
+			name: 'Kinky Sex',
+			url: 'https://www.legalporno.com/studios/kinky-sex',
+			parent_id: networksMap.legalporno,
+		},
+	];
 }

 /* eslint-disable max-len */
 exports.seed = knex => Promise.resolve()
-    .then(async () => {
-        const networks = await knex('networks').select('*');
-        const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+	.then(async () => {
+		const [networks, types] = await Promise.all([
+			knex('entities').select('*'),
+			knex('entities_types').select('*'),
+		]);

-        const studios = getStudios(networksMap);
+		const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+		const typesMap = types.reduce((acc, type) => ({ [type.type]: type.id }), {});

-        return upsert('studios', studios, 'slug', knex);
-    });
+		const studios = getStudios(networksMap).map(studio => ({
+			...studio,
+			type: typesMap.studio,
+		}));
+
+		return upsert('entities', studios, 'slug', knex);
+	});
--- a/seeds/04_media.js
+++ b/seeds/04_media.js
@ -591,7 +591,7 @@ const tagPosters = [
 	['anal-creampie', 1, 'Aleska Diamond in "Aleska Wants More" for Asshole Fever'],
 	['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'],
 	['asian', 0, 'Alina Li in "Slut Puppies 8" for Jules Jordan'],
-	['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],
+	['atm', 6, 'Jane Wilde in "Teen Anal" for Evil Angel'],
 	['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
 	['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
 	['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
@ -659,6 +659,7 @@ const tagPhotos = [
 	['airtight', 2, 'Dakota Skye in "Dakota Goes Nuts" for ArchAngel'],
 	['airtight', 3, 'Anita Bellini in "Triple Dick Gangbang" for Hands On Hardcore (DDF Network)'],
 	['anal-creampie', 0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
+	['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],
 	['atm', 0, 'Roxy Lips in "Under Her Coat" for 21 Naturals'],
 	['atm', 3, 'Natasha Teen in "Work That Ass!" for Her Limit'],
 	['asian', 'poster', 'Vina Sky in "Slut Puppies 15" for Jules Jordan'],
--- a/src/argv.js
+++ b/src/argv.js
@ -22,7 +22,7 @@ const { argv } = yargs
 	.option('sites', {
 		describe: 'Sites to scrape (overrides configuration)',
 		type: 'array',
-		alias: 'site',
+		alias: 'entities',
 	})
 	.option('actors', {
 		describe: 'Scrape actors by name or slug',
--- a/src/updates.js
+++ b/src/updates.js
@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const include = require('./utils/argv-include')(argv);
 const scrapers = require('./scrapers/scrapers');
-const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./sites');
+const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');

 const afterDate = (() => {
 	if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
@ -218,9 +218,9 @@ async function scrapeNetworkParallel(network) {
 }

 async function fetchUpdates() {
-	const includedNetworks = argv.sites || argv.networks
-		? await fetchSitesFromArgv()
-		: await fetchSitesFromConfig();
+	const includedNetworks = argv.sites || argv.networks || argv.from
+		? await fetchEntitiesFromArgv()
+		: await fetchEntitiesFromConfig();

 	const scrapedNetworks = await Promise.map(
 		includedNetworks,