diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 5788f5b5..c69e7b57 100644 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -850,10 +850,22 @@ const aliases = [ name: 'anilingus', for: 'ass-eating', }, + { + name: 'analingus', + for: 'ass-eating', + }, + { + name: 'analingus male', + for: 'ass-eating', + }, { name: 'asians', for: 'asian', }, + { + name: 'asian female', + for: 'asian', + }, { name: 'anal fingering', for: 'ass-fingering', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index de190946..9fcf96fa 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2075,6 +2075,9 @@ const sites = [ url: 'https://seehimfuck.com', tags: ['male-focus'], network: 'hussiepass', + parameters: { + tour: true, + }, }, { slug: 'interracialpovs', @@ -2082,13 +2085,20 @@ const sites = [ url: 'https://www.interracialpovs.com', tags: ['interracial', 'pov'], network: 'hussiepass', + parameters: { + tour: true, + }, }, { slug: 'povpornstars', name: 'POV Pornstars', - url: 'https://www.povpornstars.com', + url: 'http://www.povpornstars.com', tags: ['pov'], network: 'hussiepass', + parameters: { + latest: 'http://www.povpornstars.com/tour/categories/movies_%d_d.html', + tour: true, + }, }, // HUSH PASS { diff --git a/src/releases.js b/src/releases.js index abb4b22d..6c153c4b 100644 --- a/src/releases.js +++ b/src/releases.js @@ -423,6 +423,11 @@ async function updateReleasesSearch(releaseIds) { } async function storeRelease(release, batchId) { + if (!release.entryId) { + logger.warn(`Missing entry ID, unable to store ${release.url}`); + return null; + } + const existingRelease = await knex('releases') .where({ entry_id: release.entryId, @@ -473,11 +478,11 @@ async function storeReleases(releases) { try { const releaseWithChannelSite = await attachChannelSite(release); const releaseWithStudio = await attachStudio(releaseWithChannelSite); - const { id, slug } = await storeRelease(releaseWithStudio, batchId); + const storedRelease = await storeRelease(releaseWithStudio, batchId); - return { - id, - slug, + return storedRelease && { + id: storedRelease.id, + slug: storedRelease.slug, ...releaseWithChannelSite, }; } catch (error) { @@ -487,7 +492,7 @@ async function storeReleases(releases) { } }, { concurrency: 10, - }).filter(release => release); + }).filter(Boolean); logger.info(`Stored ${storedReleases.length} new releases`); diff --git a/src/scrapers/hush.js b/src/scrapers/hush.js index fd99206b..1cc6a665 100644 --- a/src/scrapers/hush.js +++ b/src/scrapers/hush.js @@ -6,6 +6,10 @@ const knex = require('../knex'); const { get, geta, fd } = require('../utils/q'); const slugify = require('../utils/slugify'); +function deriveEntryId(release) { + return `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; +} + function extractPoster(posterPath, site, baseRelease) { if (posterPath && !/400.jpg/.test(posterPath)) { const poster = `${site.parameters?.media || site.url}${posterPath}`; @@ -60,9 +64,6 @@ function scrapeLatestT1(scenes, site) { release.date = qd('.more-info-div', 'MMM D, YYYY'); release.duration = ql('.more-info-div'); - // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; - release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; - const posterPath = q('.img-div img', 'src0_1x') || qi('img.video_placeholder'); if (posterPath) { @@ -75,6 +76,27 @@ function scrapeLatestT1(scenes, site) { ]; } + // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; + release.entryId = deriveEntryId(release); + + return release; + }); +} + +function scrapeLatestTour(scenes) { + return scenes.map(({ q, qa, qu, qd, qi }) => { + const release = {}; + + release.title = q('h4 a', true); + release.url = qu('a'); + release.date = qd('.tour_update_models + span', 'YYYY-MM-DD'); + + release.actors = qa('.tour_update_models a', true); + + release.poster = qi('a img'); + + release.entryId = deriveEntryId(release); + return release; }); } @@ -112,9 +134,6 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease, cha release.date = qd('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/); release.duration = ql('.update-info-row:nth-child(2)'); - // release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1]; - release.entryId = `${slugify(fd(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; - release.actors = qa('.models-list-thumbs a').map(el => ({ name: q(el, 'span', true), avatar: [ @@ -148,6 +167,35 @@ function scrapeSceneT1({ html, q, qa, qd, ql, qtx }, site, url, baseRelease, cha } } + // release.entryId = q('.player-thumb img', 'id')?.match(/set-target-(\d+)/)[1]; + release.entryId = deriveEntryId(release); + + return release; +} + +function scrapeSceneTour({ html, q, qd, qa, qis }, site, url) { + const release = { url }; + + release.title = q('.update_title, .video-title', true); + release.description = q('.latest_update_description, .video-summary', true); + + const date = qd('.availdate', 'YYYY-MM-DD'); + if (date) release.date = date; + + release.actors = qa('.update_block_info .tour_update_models a, .video-model .tour_update_models a', true); + release.tags = qa('.update_tags a, .tour_update_tags a', true); + + const [photo, poster, ...photos] = qis('.update_image img:not(.play_icon_overlay)'); + if (poster || photo) release.poster = poster || photo; + if ((photo && poster) || photos) release.photos = poster ? [photo, ...photos] : photos; // don't use first photo when already used as fallback poster + + if (release.date) release.entryId = deriveEntryId(release); + + const trailerCode = q('.update_image a', 'onclick'); + const trailerPath = trailerCode?.match(/tload\('(.*)'\)/)?.[1] || html.match(/\/trailer\/.*\.mp4/)?.[0]; + if (trailerPath && /^http/.test(trailerPath)) release.trailer = { src: trailerPath }; + else if (trailerPath) release.trailer = { src: `${site.parameters?.media || site.url}${trailerPath}` }; + return release; } @@ -164,11 +212,13 @@ async function fetchLatest(site, page = 1) { || (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`) || `${site.url}/categories/movies_${page}_d.html`; - const qLatest = await geta(url, '.modelfeature, .item-video'); + const qLatest = await geta(url, '.modelfeature, .item-video, .updateItem'); if (!qLatest) return null; + if (site.parameters?.t1) return scrapeLatestT1(qLatest, site); + if (site.parameters?.tour) return scrapeLatestTour(qLatest, site); - return site.parameters?.t1 ? scrapeLatestT1(qLatest, site) : scrapeLatest(qLatest, site); + return scrapeLatest(qLatest, site); } async function fetchScene(url, site, baseRelease, beforeFetchLatest) { @@ -177,7 +227,10 @@ async function fetchScene(url, site, baseRelease, beforeFetchLatest) { if (!qScene) return null; - return site.parameters?.t1 ? scrapeSceneT1(qScene, site, url, baseRelease, channelRegExp) : scrapeScene(qScene, site, url, baseRelease); + if (site.parameters?.t1) return scrapeSceneT1(qScene, site, url, baseRelease, channelRegExp); + if (site.parameters?.tour) return scrapeSceneTour(qScene, site, url, baseRelease); + + return scrapeScene(qScene, site, url, baseRelease); } module.exports = {