boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactored update scraper into new module.

This commit is contained in:
ThePendulum 2020-03-14 02:56:28 +01:00
parent 4fc0053bd9
commit c8ebe7892a
6 changed files with 263 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
config/default.js
View File

@ -12,9 +12,44 @@ module.exports = {
// include: [],
// exclude: [],
exclude: [
['21sextreme', [
// no longer updated
'mightymistress',
'dominatedgirls',
'homepornreality',
'peeandblow',
'cummingmatures',
'mandyiskinky',
'speculumplays',
'creampiereality',
]],
['blowpass', ['sunlustxxx']],
['ddfnetwork', [
'fuckinhd',
'bustylover',
]],
['famedigital', [
'daringsex',
'lowartfilms',
]],
['pornpros', [
'milfhumiliation',
'humiliated',
'flexiblepositions',
'publicviolations',
'amateurviolations',
'squirtdisgrace',
'cumdisgrace',
'webcamhackers',
'collegeteens',
]],
['score', [
'bigboobbundle',
'milfbundle',
'pornmegaload',
'scorelandtv',
'scoretv',
]],
],
profiles: [
[

3
migrations/20190325001339_releases.js
View File

@ -162,10 +162,9 @@ exports.up = knex => Promise.resolve()
table.integer('priority', 3)
.defaultTo(0);
table.boolean('show')
.defaultTo(true);
table.boolean('scrape')
.defaultTo(true);
table.datetime('created_at')
.defaultTo(knex.fn.now());

29
seeds/02_sites.js
View File

@ -81,7 +81,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'dominatedgirls',
@ -91,7 +90,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'homepornreality',
@ -101,7 +99,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'peeandblow',
@ -111,7 +108,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'cummingmatures',
@ -121,7 +117,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'mandyiskinky',
@ -131,7 +126,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'speculumplays',
@ -141,7 +135,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
{
slug: 'creampiereality',
@ -151,7 +144,6 @@ const sites = [
parameters: {
scene: 'https://21sextreme.com/en/video',
},
scrape: false, // no longer updated
},
// 21SEXTURY
{
@ -1098,7 +1090,6 @@ const sites = [
url: 'https://www.sunlustxxx.com',
description: '',
network: 'blowpass',
scrape: false,
show: true, // site offline, use only for indexing old scenes
},
// BRAZZERS
@ -1467,8 +1458,6 @@ const sites = [
description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
network: 'ddfnetwork',
parameters: { native: true },
scrape: false,
show: true, // appears to be re-releases only
},
{
slug: 'bustylover',
@ -1476,8 +1465,6 @@ const sites = [
url: 'https://bustylover.com',
network: 'ddfnetwork',
parameters: { native: true },
scrape: false,
show: true, // appears to be re-releases only
},
// DIGITAL PLAYGROUND
{
@ -1815,7 +1802,6 @@ const sites = [
description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
network: 'famedigital',
parameters: { api: true },
scrape: false,
show: false, // no data sources
},
{
@ -4141,14 +4127,12 @@ const sites = [
url: 'https://milfhumiliation.com',
network: 'pornpros',
tags: ['milf'],
scrape: false,
},
{
name: 'Humiliated',
slug: 'humiliated',
url: 'https://humiliated.com',
network: 'pornpros',
scrape: false,
},
{
name: 'Flexible Positions',
@ -4158,7 +4142,6 @@ const sites = [
parameters: {
network: true,
},
scrape: false,
},
{
name: 'Public Violations',
@ -4168,41 +4151,35 @@ const sites = [
parameters: {
network: true,
},
scrape: false,
},
{
name: 'Amateur Violations',
slug: 'amateurviolations',
url: 'https://amateurviolations.com',
network: 'pornpros',
scrape: false,
},
{
name: 'Squirt Disgrace',
slug: 'squirtdisgrace',
url: 'https://squirtdisgrace.com',
network: 'pornpros',
scrape: false,
},
{
name: 'Cum Disgrace',
slug: 'cumdisgrace',
url: 'https://cumdisgrace.com',
network: 'pornpros',
scrape: false,
},
{
name: 'Webcam Hackers',
slug: 'webcamhackers',
url: 'https://webcamhackers.com',
network: 'pornpros',
scrape: false,
},
{
name: 'College Teens',
slug: 'collegeteens',
network: 'pornpros',
scrape: false,
},
// PRIVATE
{
@ -4838,7 +4815,6 @@ const sites = [
slug: 'bigboobbundle',
url: 'https://www.bigboobbundle.com',
network: 'score',
scrape: false,
show: false, // all content appears to be on subsites
},
{
@ -5132,7 +5108,6 @@ const sites = [
url: 'https://www.milfbundle.com',
network: 'score',
show: false,
scrape: false,
},
{
name: 'Teaming Cock',
@ -5200,7 +5175,6 @@ const sites = [
url: 'https://www.pornmegaload.com',
network: 'score',
show: false,
scrape: false,
},
{
name: 'SaRennas World',
@ -5238,7 +5212,6 @@ const sites = [
url: 'https://www.scorepass.com/scorelandtv',
network: 'score',
priority: 1,
scrape: false,
show: false, // appears to be streaming service for other sites
},
{
@ -5247,7 +5220,6 @@ const sites = [
url: 'https://www.scoretv.tv',
network: 'score',
priority: 1,
scrape: false,
show: false, // similar to or same as Scoreland TV
},
{
@ -6114,7 +6086,6 @@ exports.seed = knex => Promise.resolve()
parameters: site.parameters,
network_id: networksMap[site.network],
priority: site.priority,
scrape: site.scrape,
show: site.show,
}));

19
src/app.js
View File

@ -1,9 +1,23 @@
'use strict';
// const knex = require('./knex');
const argv = require('./argv');
const knex = require('./knex');
const initServer = require('./web/server');
const knex = require('./knex');
const fetchUpdates = require('./fetch-updates');
async function init() {
if (argv.server) {
await initServer();
return;
}
await fetchUpdates();
knex.destroy();
}
/*
const scrapeSites = require('./scrape-sites');
const { scrapeScenes, scrapeMovies, deepFetchReleases } = require('./scrape-releases');
const { storeReleases, updateReleasesSearch } = require('./releases');
@ -16,7 +30,7 @@ if (process.env.NODE_ENV === 'development') {
async function init() {
if (argv.scene) {
await scrapeScenes(argv.scene);
}
if (argv.movie) {
await scrapeMovies(argv.movie);
@ -52,5 +66,6 @@ async function init() {
knex.destroy();
}
*/
module.exports = init;

210
src/fetch-updates.js Normal file
View File

@ -0,0 +1,210 @@
'use strict';
const Promise = require('bluebird');
const moment = require('moment');
const argv = require('./argv');
const logger = require('./logger')(__filename);
const knex = require('./knex');
const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers');
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./sites');
const afterDate = (() => {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
// using date
return moment
.utc(argv.after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
.toDate();
}
// using time distance (e.g. "1 month")
return moment
.utc()
.subtract(...argv.after.split(' '))
.toDate();
})();
async function extractUniqueReleases(latestReleases, accReleases) {
const latestReleaseEntryIds = latestReleases.map(release => release.entryId);
const duplicateReleases = await knex('releases')
.whereIn('entry_id', latestReleaseEntryIds);
// add entry IDs of accumulated releases to prevent an infinite loop
// when one page contains the same release as the previous
const duplicateReleaseEntryIds = new Set(duplicateReleases
.map(release => String(release.entry_id))
.concat(accReleases.map(release => String(release.entryId))));
const uniqueReleases = latestReleases
.filter(release => !duplicateReleaseEntryIds.has(String(release.entryId)));
return uniqueReleases;
}
function getNextPage(uniqueReleases, pageAccReleases, oldestReleaseOnPage) {
if (uniqueReleases === 0) {
return false;
}
if (argv.last && pageAccReleases.length < argv.last) {
// request for last N releases not yet satisfied
return true;
}
if (oldestReleaseOnPage && moment(oldestReleaseOnPage.date).isAfter(afterDate)) {
// oldest release on page is newer than the specified date cut-off
return true;
}
// dates missing, and limit for scenes without dates not yet reached
return pageAccReleases.length <= argv.nullDateLimit;
}
async function scrapeLatestReleases(scraper, site, preData) {
if (!scraper.fetchLatest) {
return [];
}
const scrapePage = async (page = 1, accReleases = []) => {
const latestReleases = await scraper.fetchLatest(site, page, preData, include);
if (!Array.isArray(latestReleases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${site.name}' (${site.network.name})`);
return accReleases;
}
if (latestReleases.length === 0) {
// scraper successfully requested releases, but found none
return accReleases;
}
const latestReleasesWithSite = latestReleases.map(release => ({ ...release, site: release.site || site })); // attach site release is assigned to when stored
const oldestReleaseOnPage = latestReleases.sort((releaseA, releaseB) => releaseB.date - releaseA.date).slice(-1)[0];
const uniqueReleases = await extractUniqueReleases(latestReleasesWithSite, accReleases);
const pageAccReleases = accReleases.concat(uniqueReleases);
logger.verbose(`Scraped '${site.name}' (${site.network.name}) page ${page}, found ${uniqueReleases.length} unique releases`);
if (getNextPage(uniqueReleases, pageAccReleases, oldestReleaseOnPage)) {
return scrapePage(page + 1, accReleases.concat(uniqueReleases));
}
if (argv.last) {
return pageAccReleases.slice(0, argv.last);
}
if (oldestReleaseOnPage) {
const recentReleases = uniqueReleases
.filter(release => moment(release.date).isAfter(afterDate));
return accReleases.concat(recentReleases);
}
return pageAccReleases.slice(0, argv.nullDateLimit);
};
return scrapePage(1, []);
}
async function scrapeUpcomingReleases(scraper, site, preData) {
if (!scraper.fetchUpcoming) {
return [];
}
try {
const upcomingReleases = await scraper.fetchUpcoming(site, 1, preData, include);
if (upcomingReleases) {
return upcomingReleases.map(release => ({
...release,
site,
upcoming: true,
}));
}
} catch (error) {
logger.warn(`Failed to scrape upcoming releases for '${site.slug}' (${site.network.slug})`);
}
return [];
}
async function scrapeSiteReleases(scraper, site, preData) {
const [latestReleases, upcomingReleases] = await Promise.all([
argv.latest
? scrapeLatestReleases(scraper, site, preData)
: [],
argv.upcoming
? scrapeUpcomingReleases(scraper, site, preData)
: [],
]);
return [...latestReleases, ...upcomingReleases];
}
async function scrapeSite(site, accSiteReleases) {
const scraper = scrapers.releases[site.slug]
|| scrapers.releases[site.network.slug]
|| scrapers.releases[site.network.parent?.slug];
if (!scraper) {
logger.warn(`No scraper found for '${site.name}' (${site.network.name})`);
return [];
}
try {
const beforeFetchLatest = await scraper.beforeFetchLatest?.(site);
const siteReleases = await scrapeSiteReleases(scraper, site, {
accSiteReleases,
beforeFetchLatest,
});
return siteReleases.map(release => ({ ...release, site }));
} catch (error) {
logger.error(`Failed to scrape releases from ${site.name} using ${scraper.slug}: ${error.message}`);
return [];
}
}
async function scrapeNetworkSequential(network) {
return Promise.reduce(
network.sites,
async (chain, site) => {
const accSiteReleases = await chain;
const siteReleases = await scrapeSite(site, network, accSiteReleases);
return accSiteReleases.concat(siteReleases);
},
Promise.resolve([]),
);
}
async function scrapeNetworkParallel(network) {
return Promise.map(
network.sites,
async site => scrapeSite(site, network),
{ concurrency: 3 },
);
}
async function fetchUpdates() {
const includedNetworks = argv.sites || argv.networks
? await fetchSitesFromArgv()
: await fetchSitesFromConfig();
const scrapedNetworks = await Promise.map(
includedNetworks,
async network => (network.parameters?.sequential
? scrapeNetworkSequential(network)
: scrapeNetworkParallel(network)),
{ concurrency: 5 },
);
return scrapedNetworks;
}
module.exports = fetchUpdates;

2
src/sites.js
View File

@ -113,7 +113,6 @@ async function fetchSitesFromArgv() {
)
.whereIn('sites.slug', argv.sites || [])
.orWhereIn('networks.slug', argv.networks || [])
.where('sites.scrape', true)
.leftJoin('networks', 'sites.network_id', 'networks.id');
const curatedSites = await curateSites(rawSites, true);
@ -132,7 +131,6 @@ async function fetchSitesFromConfig() {
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.where('sites.scrape', true)
.where((builder) => {
if (config.include) {
builder