diff --git a/public/img/logos/hushpass/favicon.png b/public/img/logos/hushpass/favicon.png new file mode 100644 index 00000000..bb2ebfd5 Binary files /dev/null and b/public/img/logos/hushpass/favicon.png differ diff --git a/public/img/logos/interracialpass/favicon.png b/public/img/logos/interracialpass/favicon.png new file mode 100644 index 00000000..44cdb825 Binary files /dev/null and b/public/img/logos/interracialpass/favicon.png differ diff --git a/seeds/02_sites.js b/seeds/02_sites.js index ea6e6140..de190946 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2189,13 +2189,14 @@ const sites = [ }, }, { - slug: 'buttnakedinstreets', + slug: 'buttnakedinthestreets', name: 'Butt Naked In The Streets', url: 'https://buttnakedinthestreets.com', network: 'hushpass', parameters: { latest: 'https://hushpass.com/t1/categories/ButtNakedInStreets_%d_d.html', media: 'https://hushpass.com', + match: 'Butt Naked In Streets', t1: true, }, }, @@ -2264,6 +2265,89 @@ const sites = [ t1: true, }, }, + { + slug: '2bigtobetrue', + name: '2 Big To Be True', + url: 'https://www.2bigtobetrue.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/2-big-to-be-true_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'abominableblackman', + name: 'Abominable Black Man', + url: 'https://www.abominableblackman.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/abominable-black-man_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'bootyannihilation', + name: 'Booty Annihilation', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'daddysworstnightmare', + name: 'Daddy\'s Worst Nightmare', + url: 'https://www.daddysworstnightmare.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/daddys-worst-nightmare_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'monstercockfuckfest', + name: 'Monster Cock Fuck Fest', + url: 'https://www.monstercockfuckfest.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/monster-cock-fuck-fest_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'mymomsfuckingblackzilla', + name: 'My Mom\'s Fucking Blackzilla', + url: 'https://www.mymomsfuckingblackzilla.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/my-moms-fucking-blackzilla_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, + { + slug: 'mywifesfirstmonstercock', + name: 'My Wife\'s First Monster Cock', + url: 'https://www.mywifesfirstmonstercock.com/', + tags: ['interracial'], + network: 'interracialpass', + parameters: { + latest: 'https://www.interracialpass.com/t1/categories/my-wifes-first-monster-cock_%d_d.html', + media: 'https://www.interracialpass.com', + t1: true, + }, + }, // INSEX { slug: 'sexuallybroken', diff --git a/src/scrape-sites.js b/src/scrape-sites.js index 18440ca1..91be1fab 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -37,12 +37,12 @@ async function findDuplicateReleaseIds(latestReleases, accReleases) { .concat(accReleases.map(release => String(release.entryId)))); } -async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAfterDate(), accReleases = [], page = argv.page) { +async function scrapeUniqueReleases(scraper, site, beforeFetchLatest, afterDate = getAfterDate(), accReleases = [], page = argv.page) { if (!argv.latest || !scraper.fetchLatest) { return []; } - const latestReleases = await scraper.fetchLatest(site, page, preflight); + const latestReleases = await scraper.fetchLatest(site, page, beforeFetchLatest); if (!Array.isArray(latestReleases)) { logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`); @@ -73,7 +73,7 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft || (argv.last && accReleases.length + uniqueReleases.length < argv.last)) ) { // oldest release on page is newer that specified date range, or latest count has not yet been met, fetch next page - return scrapeUniqueReleases(scraper, site, preflight, afterDate, accReleases.concat(uniqueReleases), page + 1); + return scrapeUniqueReleases(scraper, site, beforeFetchLatest, afterDate, accReleases.concat(uniqueReleases), page + 1); } if (argv.last && uniqueReleases.length >= argv.last) { @@ -87,9 +87,9 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft return accReleases.concat(uniqueReleases).slice(0, argv.nullDateLimit); } -async function scrapeUpcomingReleases(scraper, site, preflight) { +async function scrapeUpcomingReleases(scraper, site, beforeFetchLatest) { if (argv.upcoming && scraper.fetchUpcoming) { - const upcomingReleases = await scraper.fetchUpcoming(site, 1, preflight); + const upcomingReleases = await scraper.fetchUpcoming(site, 1, beforeFetchLatest); return upcomingReleases ? upcomingReleases.map(release => ({ ...release, site, upcoming: true })) @@ -99,11 +99,11 @@ async function scrapeUpcomingReleases(scraper, site, preflight) { return []; } -async function deepFetchReleases(baseReleases, preflight) { +async function deepFetchReleases(baseReleases, beforeFetchLatest) { return Promise.map(baseReleases, async (release) => { if (release.url || (release.path && release.site)) { try { - const fullRelease = await scrapeRelease(release.url, release, 'scene', preflight); + const fullRelease = await scrapeRelease(release.url, release, 'scene', beforeFetchLatest); if (fullRelease) { return { @@ -133,11 +133,11 @@ async function deepFetchReleases(baseReleases, preflight) { } async function scrapeSiteReleases(scraper, site) { - const preflight = await scraper.preflight?.(site); + const beforeFetchLatest = await scraper.beforeFetchLatest?.(site); const [newReleases, upcomingReleases] = await Promise.all([ - scrapeUniqueReleases(scraper, site, preflight), // fetch basic release info from scene overview - scrapeUpcomingReleases(scraper, site, preflight), // fetch basic release info from upcoming overview + scrapeUniqueReleases(scraper, site, beforeFetchLatest), // fetch basic release info from scene overview + scrapeUpcomingReleases(scraper, site, beforeFetchLatest), // fetch basic release info from upcoming overview ]); if (argv.upcoming) { @@ -148,7 +148,7 @@ async function scrapeSiteReleases(scraper, site) { if (argv.deep) { // follow URL for every release - return deepFetchReleases(baseReleases, preflight); + return deepFetchReleases(baseReleases, beforeFetchLatest); } return baseReleases; diff --git a/src/scrapers/assylum.js b/src/scrapers/assylum.js index fc4ac03e..5ed37d30 100644 --- a/src/scrapers/assylum.js +++ b/src/scrapers/assylum.js @@ -117,8 +117,8 @@ async function fetchLatest(site, page = 1, models) { return qLatest && scrapeLatest(qLatest, site, models); } -async function fetchScene(url, site, release, preflight) { - const models = preflight || await fetchModels(site); +async function fetchScene(url, site, release, beforeFetchLatest) { + const models = beforeFetchLatest || await fetchModels(site); const qScene = await get(url); return qScene && scrapeScene(qScene, url, site, models); @@ -127,5 +127,5 @@ async function fetchScene(url, site, release, preflight) { module.exports = { fetchLatest, fetchScene, - preflight: fetchModels, + beforeFetchLatest: fetchModels, }; diff --git a/src/scrapers/hush.js b/src/scrapers/hush.js index cbbd6a17..fd99206b 100644 --- a/src/scrapers/hush.js +++ b/src/scrapers/hush.js @@ -2,6 +2,7 @@ const util = require('util'); +const knex = require('../knex'); const { get, geta, fd } = require('../utils/q'); const slugify = require('../utils/slugify'); @@ -22,7 +23,7 @@ function extractPoster(posterPath, site, baseRelease) { return [posterSources, []]; } - return [null, []]; + return [baseRelease?.poster || null, []]; } function scrapeLatest(scenes, site) { @@ -59,8 +60,8 @@ function scrapeLatestT1(scenes, site) { release.date = qd('.more-info-div', 'MMM D, YYYY'); release.duration = ql('.more-info-div'); - release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1] - || `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; + // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; + release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; const posterPath = q('.img-div img', 'src0_1x') || qi('img.video_placeholder'); @@ -102,7 +103,7 @@ function scrapeScene({ html, q, qa, qd, ql }, site, url, baseRelease) { return release; } -function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease) { +function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease, channelRegExp) { const release = { url }; release.title = q('.trailer-section-head .section-title', true); @@ -111,8 +112,8 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease) { release.date = qd('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.duration = ql('.update-info-row:nth-child(2)'); - release.entryId = q('.player-thumb img', 'id').match(/set-target-(\d+)/)[1] - || `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; + // release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1]; + release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; release.actors = qa('.models-list-thumbs a').map(el => ({ name: q(el, 'span', true), @@ -136,8 +137,8 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease) { const stars = q('.update-rating', true).match(/\d.\d/)?.[0]; if (stars) release.stars = Number(stars); - if (site.slug === 'hushpass') { - const channel = release.tags.find(tag => /Shot Her First|WhiteZilla|Frat House Fuck Fest|Freaky First Timers|MILF Invaders|Housewives Need Cash|Bubble Butt Bonanza|Suburban Sex Party|Butt Naked In Streets/i.test(tag)); + if (channelRegExp) { + const channel = release.tags.find(tag => channelRegExp.test(tag)); if (channel) { release.channel = { @@ -150,6 +151,14 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease) { return release; } +async function getChannelRegExp(site) { + if (!['hushpass', 'interracialpass'].includes(site.network.slug)) return null; + + const sites = await knex('sites').where('network_id', site.network.id); + + return new RegExp(sites.map(channel => channel.parameters?.match || channel.name).join('|'), 'i'); +} + async function fetchLatest(site, page = 1) { const url = (site.parameters?.latest && util.format(site.parameters.latest, page)) || (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`) @@ -162,16 +171,17 @@ async function fetchLatest(site, page = 1) { return site.parameters?.t1 ? scrapeLatestT1(qLatest, site) : scrapeLatest(qLatest, site); } -async function fetchScene(url, site, baseRelease) { +async function fetchScene(url, site, baseRelease, beforeFetchLatest) { + const channelRegExp = beforeFetchLatest || await getChannelRegExp(site); const qScene = await get(url); if (!qScene) return null; - return site.parameters?.t1 ? scrapeSceneT1(qScene, site, url, baseRelease) : scrapeScene(qScene, site, url, baseRelease); + return site.parameters?.t1 ? scrapeSceneT1(qScene, site, url, baseRelease, channelRegExp) : scrapeScene(qScene, site, url, baseRelease); } module.exports = { - // preflight, + beforeFetchLatest: getChannelRegExp, fetchLatest, fetchScene, };