Added parent-child relations to network, showing parent network in sidebar. Added Burning Angel using Gamma API.

This commit is contained in:
ThePendulum 2020-02-20 02:35:23 +01:00
parent 6b358d74db
commit 377970f874
30 changed files with 3174 additions and 913 deletions

View File

@ -7,10 +7,10 @@
<div
class="network"
:class="{ nosites: sites.length === 0 }"
:class="{ nosites: sites.length === 0 && networks.length === 0 }"
>
<div
v-show="sites.length > 0"
v-show="sites.length > 0 || networks.length > 0"
class="sidebar"
:class="{ expanded }"
>
@ -37,9 +37,21 @@
:sites="sites"
:class="{ expanded }"
/>
<Network
v-for="childNetwork in networks"
:key="`network-${childNetwork.id}`"
:network="childNetwork"
/>
<Network
v-if="network.parent"
:network="network.parent"
class="parent"
/>
</div>
<template v-if="sites.length > 0">
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="!expanded"
class="expand expand-sidebar noselect"
@ -55,7 +67,7 @@
<div
class="header"
:class="{ hideable: sites.length > 0 }"
:class="{ hideable: sites.length > 0 || networks.length > 0 }"
>
<a
v-tooltip.bottom="`Go to ${network.url}`"
@ -72,7 +84,7 @@
</div>
<div class="content-inner">
<template v-if="sites.length > 0">
<template v-if="sites.length > 0 || networks.length > 0">
<span
v-show="expanded"
class="expand collapse-header noselect"
@ -108,6 +120,7 @@
import FilterBar from '../header/filter-bar.vue';
import Releases from '../releases/releases.vue';
import Sites from '../sites/sites.vue';
import Network from '../tile/network.vue';
async function fetchNetwork() {
this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug);
@ -119,6 +132,7 @@ async function fetchNetwork() {
}));
}
this.networks = this.network.networks;
this.sites = this.network.sites
.filter(site => !site.independent);
@ -135,11 +149,13 @@ export default {
FilterBar,
Releases,
Sites,
Network,
},
data() {
return {
network: null,
sites: [],
networks: [],
studios: [],
releases: [],
pageTitle: null,
@ -212,6 +228,15 @@ export default {
filter: $logo-highlight;
}
.parent {
display: inline-block;
height: 3rem;
:before {
content: 'Part of';
}
}
.header {
width: 100%;
display: flex;

View File

@ -38,6 +38,7 @@ export default {
.sites {
overflow: hidden;
display: flex;
flex-grow: 1;
&.compact:not(.expanded) {
flex-direction: row;

View File

@ -27,14 +27,15 @@ export default {
@import 'theme';
.tile {
height: 6rem;
background: $profile;
display: flex;
flex-shrink: 0;
flex-direction: column;
align-items: center;
box-sizing: border-box;
padding: .5rem 1rem;
box-shadow: 0 0 3px rgba(0, 0, 0, .25);
height: 100%;
text-align: center;
}
@ -44,7 +45,7 @@ export default {
.logo {
width: 100%;
height: 5rem;
height: 100%;
color: $text;
display: flex;
align-items: center;

View File

@ -27,15 +27,16 @@ export default {
@import 'theme';
.tile {
height: 6rem;
background: $tile;
display: flex;
flex-shrink: 0;
flex-direction: column;
align-items: center;
box-sizing: border-box;
padding: .5rem 1rem;
border-radius: .25rem;
box-shadow: 0 0 3px rgba(0, 0, 0, .25);
height: 100%;
text-align: center;
}

View File

@ -57,9 +57,12 @@ function curateNetwork(network, releases) {
name: network.name,
slug: network.slug,
url: network.url,
networks: [],
};
if (network.parent) curatedNetwork.parent = curateNetwork(network.parent);
if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork));
if (network.studios) curatedNetwork.studios = network.studios;
if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release));

View File

@ -18,6 +18,14 @@ function initNetworksActions(store, _router) {
name
slug
url
networks: childNetworks(
orderBy: NAME_ASC,
) {
id
name
slug
url
}
sites(
orderBy: [PRIORITY_DESC, NAME_ASC],
filter: {
@ -45,12 +53,23 @@ function initNetworksActions(store, _router) {
slug
url
}
parent {
id
name
slug
url
}
}
releases(
first: $limit,
orderBy: $orderBy,
filter: {
site: { network: { slug: { equalTo: $networkSlug } } }
site: {
or: [
{ network: { slug: { equalTo: $networkSlug } } },
{ network: { parent: { slug: { equalTo: $networkSlug } } } }
]
}
date: {
lessThan: $before,
greaterThan: $after,

View File

@ -22,7 +22,7 @@ module.exports = {
'famedigital',
],
[
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable) and Wicked have their own assets
// Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
'xempire',
'blowpass',
],
@ -40,6 +40,7 @@ module.exports = {
'transangels',
],
'wicked',
'burningangel',
'brazzers',
'milehighmedia',
[
@ -93,7 +94,7 @@ module.exports = {
'freeonesLegacy',
],
fetchAfter: [1, 'week'],
nullDateLimit: 10,
nullDateLimit: 3,
media: {
path: './media',
thumbnailSize: 320, // width for 16:9 will be exactly 576px

View File

@ -108,6 +108,10 @@ exports.up = knex => Promise.resolve()
table.text('description');
table.json('parameters');
table.integer('parent_id', 12)
.references('id')
.inTable('networks');
table.string('slug', 32)
.unique();

3807
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -35,12 +35,12 @@
"author": "Niels Simenon",
"license": "ISC",
"devDependencies": {
"@babel/cli": "^7.7.5",
"@babel/core": "^7.7.5",
"@babel/plugin-proposal-optional-chaining": "^7.7.5",
"@babel/preset-env": "^7.7.6",
"@babel/register": "^7.7.4",
"autoprefixer": "^9.7.3",
"@babel/cli": "^7.8.4",
"@babel/core": "^7.8.4",
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
"@babel/preset-env": "^7.8.4",
"@babel/register": "^7.8.3",
"autoprefixer": "^9.7.4",
"babel-eslint": "^10.0.3",
"babel-loader": "^8.0.6",
"babel-preset-airbnb": "^3.3.2",
@ -49,50 +49,51 @@
"eslint-config-airbnb": "^17.1.1",
"eslint-config-airbnb-base": "^13.2.0",
"eslint-loader": "^2.2.1",
"eslint-plugin-import": "^2.19.1",
"eslint-plugin-import": "^2.20.1",
"eslint-plugin-jsx-a11y": "^6.2.3",
"eslint-plugin-react": "^7.17.0",
"eslint-plugin-vue": "^6.0.1",
"eslint-plugin-react": "^7.18.3",
"eslint-plugin-vue": "^6.2.1",
"eslint-watch": "^4.0.2",
"mini-css-extract-plugin": "^0.7.0",
"node-sass": "^4.13.0",
"node-sass": "^4.13.1",
"postcss-loader": "^3.0.0",
"raw-loader": "^2.0.0",
"sass-loader": "^7.3.1",
"style-loader": "^0.23.1",
"vue-loader": "^15.7.2",
"vue-template-compiler": "^2.6.10",
"webpack": "^4.41.2",
"webpack-cli": "^3.3.10"
"vue-loader": "^15.9.0",
"vue-template-compiler": "^2.6.11",
"webpack": "^4.41.6",
"webpack-cli": "^3.3.11"
},
"dependencies": {
"@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
"@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
"@tensorflow/tfjs-node": "^1.5.1",
"@tensorflow/tfjs-node": "^1.5.2",
"babel-polyfill": "^6.26.0",
"bhttp": "^1.2.4",
"bhttp": "^1.2.6",
"blake2": "^4.0.0",
"bluebird": "^3.7.2",
"body-parser": "^1.19.0",
"canvas": "^2.6.1",
"cheerio": "^1.0.0-rc.3",
"cli-confirm": "^1.0.1",
"config": "^3.2.4",
"dayjs": "^1.8.17",
"config": "^3.2.5",
"dayjs": "^1.8.20",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"express-react-views": "^0.11.0",
"face-api.js": "^0.21.0",
"fs-extra": "^7.0.1",
"graphile-utils": "^4.5.6",
"iconv-lite": "^0.5.0",
"iconv-lite": "^0.5.1",
"jsdom": "^15.2.1",
"knex": "^0.16.5",
"knex-migrate": "^1.7.4",
"longjohn": "^0.2.12",
"mime": "^2.4.4",
"moment": "^2.24.0",
"opn": "^5.5.0",
"pg": "^7.14.0",
"pg": "^7.18.1",
"postgraphile": "^4.5.5",
"postgraphile-plugin-connection-filter": "^1.1.3",
"prop-types": "^15.7.2",
@ -103,14 +104,14 @@
"source-map-support": "^0.5.16",
"template-format": "^1.2.5",
"tough-cookie": "^3.0.1",
"tty-table": "^2.8.3",
"tty-table": "^2.8.12",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.0.2",
"vue": "^2.6.10",
"vue-router": "^3.1.3",
"v-tooltip": "^2.0.3",
"vue": "^2.6.11",
"vue-router": "^3.1.5",
"vuex": "^3.1.2",
"winston": "^3.2.1",
"winston-daily-rotate-file": "^4.4.1",
"winston-daily-rotate-file": "^4.4.2",
"yargs": "^13.3.0"
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 94 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 94 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 7.6 KiB

View File

@ -1,30 +1,48 @@
/* eslint-disable max-len */
const upsert = require('../src/utils/upsert');
const parentNetworks = [
{
slug: 'gamma',
name: 'Gamma Entertainment',
url: 'https://www.gammaentertainment.com',
},
{
slug: 'mindgeek',
name: 'Mind Geek',
url: 'https://www.mindgeek.com',
description: '',
},
];
const networks = [
{
slug: '21sextury',
name: '21Sextury',
url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parent: 'gamma',
},
{
slug: '21sextreme',
name: '21Sextreme',
url: 'https://www.21sextreme.com',
description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars',
parent: 'gamma',
},
{
slug: '21naturals',
name: '21Naturals',
url: 'https://www.21naturals.com',
description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits',
parent: 'gamma',
},
{
slug: 'adulttime',
name: 'Adult Time',
url: 'https://www.adulttime.com',
description: 'Adult Time is a premium streaming service for adults! Watch adult movies, series, and channels from the top names in the industry.',
parent: 'gamma',
},
{
slug: 'assylum',
@ -36,6 +54,7 @@ const networks = [
slug: 'babes',
name: 'Babes',
url: 'https://www.babes.com',
parent: 'mindgeek',
},
{
slug: 'bang',
@ -53,6 +72,7 @@ const networks = [
name: 'Blowpass',
url: 'https://www.blowpass.com',
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
parent: 'gamma',
},
{
slug: 'brazzers',
@ -60,6 +80,13 @@ const networks = [
url: 'https://www.brazzers.com',
description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
},
{
slug: 'burningangel',
name: 'Burning Angel',
url: 'https://www.burningangel.com',
description: 'BurningAngel.com features tattoo porn with punk girls, goths, emo and the best scene girl porn online! View Joanna Angel and all of her alt pornstars in rough sex videos and hardcore porn',
parent: 'gamma',
},
{
slug: 'cherrypimps',
name: 'Cherry Pimps',
@ -77,6 +104,7 @@ const networks = [
name: 'Digital Playground',
url: 'https://www.digitalplayground.com',
description: 'DigitalPlayground.com is the leader in high quality adult blockbuster movies and award winning sex parodies that feature the most exclusive pornstars online! Adult Film Database of adult movies.',
parent: 'mindgeek',
},
{
slug: 'dogfartnetwork',
@ -89,23 +117,27 @@ const networks = [
name: 'Evil Angel',
url: 'https://evilangel.com',
description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
parent: 'gamma',
},
{
slug: 'fantasymassage',
name: 'Fantasy Massage',
url: 'https://www.fantasymassage.com',
parent: 'gamma',
},
{
slug: 'famedigital',
name: 'Fame Digital',
url: 'https://www.famedigital.com',
description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
parent: 'gamma',
},
{
slug: 'fakehub',
name: 'Fake Hub',
url: 'https://www.fakehub.com',
description: 'Wherever they go, there is porn. Hospital, Taxis, Casting… Maybe fucking to a fake cop, fake agent or fake taxi driver. And we record it all.',
parent: 'mindgeek',
},
{
slug: 'fullpornnetwork',
@ -118,6 +150,7 @@ const networks = [
name: 'Girlsway',
url: 'https://www.girlsway.com',
description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
parent: 'gamma',
},
{
slug: 'insex',
@ -129,6 +162,7 @@ const networks = [
slug: 'jayrock',
name: 'JayRock Productions',
url: 'http://jayrockcontent.com',
parent: 'gamma',
},
{
slug: 'julesjordan',
@ -158,12 +192,14 @@ const networks = [
name: 'Men',
url: 'https://www.men.com',
description: 'Check out the best gay porn site on the net with daily updates, award-winning original series, exclusive Men.com models and over 800 of the hottest guys in gay porn.',
parent: 'mindgeek',
},
{
slug: 'metrohd',
name: 'Metro HD',
url: 'https://www.metrohd.com',
description: 'Checkout MetroHD official pornsite featuring top rated pornstars and XXX videos.',
parent: 'mindgeek',
},
{
slug: 'mikeadriano',
@ -176,18 +212,14 @@ const networks = [
name: 'Mile High Media',
url: 'https://www.milehighmedia.com',
description: 'MileHighMedia.com is the only niche porn network you need! Watch lesbian sex, hardcore fucking and family porn stories with the hottest teens & MILFs!',
},
{
slug: 'mindgeek',
name: 'Mind Geek',
url: 'https://www.mindgeek.com',
description: '',
parent: 'mindgeek',
},
{
slug: 'mofos',
name: 'MOFOS',
url: 'https://www.mofos.com',
description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
parent: 'mindgeek',
},
{
slug: 'naughtyamerica',
@ -236,12 +268,14 @@ const networks = [
name: 'Pure Taboo',
url: 'https://www.puretaboo.com',
description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.',
parent: 'gamma',
},
{
slug: 'realitykings',
name: 'Reality Kings',
url: 'https://www.realitykings.com',
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
parent: 'mindgeek',
},
{
slug: 'score',
@ -260,12 +294,14 @@ const networks = [
name: 'Twistys',
url: 'https://www.twistys.com',
description: 'The hottest high quality glamour porn for over 18 years! Over 3700+ models and 46000+ scenes. TWISTYS.com',
parent: 'mindgeek',
},
{
slug: 'vivid',
name: 'Vivid',
url: 'https://www.vivid.com',
description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
parent: 'gamma',
},
{
slug: 'vixen',
@ -284,14 +320,30 @@ const networks = [
name: 'Wicked',
url: 'https://www.wicked.com',
description: 'Welcome to the new Wicked.com! Watch over 25 years of Wicked Pictures\' brand of award-winning porn for couples and women in 4k HD movies & xxx videos',
parent: 'gamma',
},
{
slug: 'xempire',
name: 'XEmpire',
url: 'https://www.xempire.com',
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
parent: 'gamma',
},
];
exports.seed = knex => Promise.resolve()
.then(async () => upsert('networks', networks, 'slug', knex));
.then(async () => {
const { inserted, updated } = await upsert('networks', parentNetworks, 'slug', knex);
const parentNetworksBySlug = [].concat(inserted, updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const networksWithParent = networks.map(network => ({
slug: network.slug,
name: network.name,
url: network.url,
description: network.description,
parameters: network.parameters,
parent_id: parentNetworksBySlug[network.parent] || null,
}));
return upsert('networks', networksWithParent, 'slug', knex);
});

View File

@ -75,7 +75,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'dominatedgirls',
@ -85,7 +85,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'homepornreality',
@ -95,7 +95,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'peeandblow',
@ -105,7 +105,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'cummingmatures',
@ -115,7 +115,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'mandyiskinky',
@ -125,7 +125,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'speculumplays',
@ -135,7 +135,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
{
slug: 'creampiereality',
@ -145,7 +145,7 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false,
scrape: false, // no longer updated
},
// 21SEXTURY
{
@ -1265,6 +1265,14 @@ const sites = [
description: "Giant black dicks paired with round asses and garnished with the tightest pussies of all colors. Butts and Blacks delivers on its name sake, only the biggest dicks rocking the thickest chicks. These round honeys can take it all in and bounce around like it's a pogo stick. Come check out these soft round asses getting the attention they deserve.",
network: 'brazzers',
},
// BURNING ANGEL
{
name: 'Burning Angel',
slug: 'burningangel',
url: 'https://www.burningangel.com',
network: 'burningangel',
parameters: { independent: true },
},
// CHERRY PIMPS
{
slug: 'cherrypimps',

View File

@ -8,6 +8,10 @@ const scrapeSites = require('./scrape-sites');
const { scrapeReleases } = require('./scrape-releases');
const { scrapeActors, scrapeBasicActors } = require('./actors');
if (process.env.NODE_ENV === 'development') {
require('longjohn'); // eslint-disable-line global-require
}
async function init() {
if (argv.scene) {
await scrapeReleases(argv.scene, null, 'scene');

View File

@ -43,7 +43,7 @@ async function createThumbnail(buffer) {
function pluckItems(items, specifiedLimit) {
const limit = specifiedLimit || config.media.limit;
if (items.length <= limit) return items;
if (!items || items.length <= limit) return items;
const plucked = [1]
.concat(
@ -73,7 +73,9 @@ async function getEntropy(buffer) {
}
async function extractItem(source) {
const res = await bhttp.get(source.src);
const res = await bhttp.get(source.src, {
responseTimeout: 10000,
});
if (res.statusCode === 200) {
const { q } = ex(res.body.toString());
@ -114,7 +116,9 @@ async function fetchItem(source, index, existingItemsBySource, domain, role, att
logger.verbose(`Fetching media item from ${source.src || source}`);
const res = await bhttp.get(source.src || source);
const res = await bhttp.get(source.src || source, {
responseTimeout: 10000,
});
if (res.statusCode === 200) {
const { pathname } = new URL(source.src || source);
@ -199,6 +203,8 @@ async function saveItems(items, domain, role) {
logger.error(`Failed to store ${domain} ${role} from ${item.source}: ${error.message}`);
return null;
}
}, {
concurrency: 20,
});
}

View File

@ -291,6 +291,8 @@ function accumulateActors(releases) {
const actorName = actor.name ? actor.name.trim() : actor.trim();
const actorSlug = slugify(actorName);
if (!actorSlug) return;
if (!acc[actorSlug]) {
acc[actorSlug] = {
name: actorName,
@ -348,8 +350,8 @@ async function storeReleaseAssets(releases) {
associateMedia(releaseCoversById, covers, 'release', 'cover'),
]);
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
await associateMedia(releasePhotosById, photos, 'release', 'photo');
// const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
// await associateMedia(releasePhotosById, photos, 'release', 'photo');
// videos take a long time, fetch last
const [trailers, teasers] = await Promise.all([

View File

@ -44,6 +44,11 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
const latestReleases = await scraper.fetchLatest(site, page, preflight);
if (!Array.isArray(latestReleases)) {
logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`);
return accReleases;
}
if (latestReleases.length === 0) {
return accReleases;
}
@ -55,7 +60,7 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
.filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
&& (argv.last || !release.date || moment(release.date).isAfter(afterDate))); // release is older than specified date limit
logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
logger.verbose(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
const uniqueReleasesWithSite = uniqueReleases.map(release => ({ ...release, site }));

View File

@ -0,0 +1,10 @@
'use strict';
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
module.exports = {
fetchLatest: fetchApiLatest,
fetchProfile: fetchApiProfile,
fetchScene,
fetchUpcoming: fetchApiUpcoming,
};

View File

@ -131,10 +131,13 @@ async function fetchLatest(site, page = 1) {
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
console.log(url);
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site);
}
return res.statusCode;
}
async function fetchScene(url, site) {

View File

@ -186,7 +186,7 @@ function scrapeUpcoming(html, site) {
const videoClass = $(element).find('.update_thumbnail div').attr('class');
const videoScript = $(element).find(`script:contains(${videoClass})`).html();
const trailer = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
const teaser = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
return {
url: null,
@ -195,8 +195,8 @@ function scrapeUpcoming(html, site) {
date,
actors,
poster,
trailer: {
src: trailer,
teaser: {
src: teaser,
},
rating: null,
site,

View File

@ -202,7 +202,7 @@ async function fetchLatest(site, page = 1) {
return scrapeAll(res.body.toString(), site);
}
return null;
return res.statusCode;
}
async function fetchScene(url, site) {

View File

@ -8,6 +8,7 @@ const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const boobpedia = require('./boobpedia');
const brazzers = require('./brazzers');
const burningangel = require('./burningangel');
const cherrypimps = require('./cherrypimps');
const ddfnetwork = require('./ddfnetwork');
const digitalplayground = require('./digitalplayground');
@ -67,6 +68,7 @@ module.exports = {
bangbros,
blowpass,
brazzers,
burningangel,
cherrypimps,
ddfnetwork,
digitalplayground,
@ -117,6 +119,7 @@ module.exports = {
boobpedia,
brattysis: nubiles,
brazzers,
burningangel,
cherrypimps,
ddfnetwork,
deeplush: nubiles,

View File

@ -132,6 +132,7 @@ async function fetchSitesFromConfig() {
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.where('sites.scrape', true)
.where((builder) => {
if (config.include) {
builder

View File

@ -7,6 +7,8 @@ function slugify(string, {
} = {}) {
const slugComponents = string.trim().toLowerCase().match(/\w+/g);
if (!slugComponents) return '';
const slug = slugComponents.reduce((acc, component, index) => {
const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`;