diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 4566989b..4ccb9947 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -763,6 +763,35 @@ const networks = [ avatars: 'https://images.mylfcdn.net/tsv4/model/profiles', }, }, + { + slug: 'pervz', + name: 'Pervz', + url: 'https://www.pervz.com', + parent: 'paperstreetmedia', + parameters: { + layout: 'search', + }, + }, + { + slug: 'swappz', + name: 'Swappz', + url: 'https://www.swappz.com', + parent: 'paperstreetmedia', + parameters: { + layout: 'search', + endpoint: 'swap_bundle', + }, + }, + { + slug: 'freeuse', + name: 'Free Use', + url: 'https://www.freeuse.com', + parent: 'paperstreetmedia', + parameters: { + layout: 'search', + endpoint: 'freeusebundle', + }, + }, { slug: 'teencoreclub', name: 'Teen Core Club', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index ce9802c1..a78051b6 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -8183,15 +8183,6 @@ const sites = [ id: 'milfbody', }, }, - { - slug: 'milftaxi', - name: 'MILF Taxi', - url: 'https://www.mylf.com/series/milf-taxi', - parent: 'mylf', - parameters: { - id: 'milf-taxi', - }, - }, { slug: 'milfty', name: 'Milfty', @@ -8767,17 +8758,6 @@ const sites = [ endpoint: 'organic-1-goide6Xo', }, }, - { - slug: 'freeusemilf', - name: 'Freeuse MILF', - url: 'https://www.freeusemilf.com', - tags: ['free-use', 'milf'], - parent: 'mylf', - parameters: { - layout: 'organic', - endpoint: 'organic-Freeusemilf-uug2tohT', - }, - }, { slug: 'hijabmylfs', name: 'Hijab Mylfs', @@ -8789,17 +8769,6 @@ const sites = [ endpoint: 'Organic-bad-aiGhaiL5', }, }, - { - slug: 'momswap', - name: 'Mom Swap', - url: 'https://www.momswap.com', - tags: ['family', 'milf'], - parent: 'mylf', - parameters: { - layout: 'organic', - endpoint: 'organic-momswap-6fkccwxhi0', - }, - }, { slug: 'mylfdom', name: 'MYLF Dom', @@ -8822,28 +8791,6 @@ const sites = [ endpoint: 'Organic-pna-OongoaF1', }, }, - { - slug: 'pervprincipal', - name: 'Perv Principal', - url: 'https://www.pervprincipal.com', - tags: ['milf'], - parent: 'mylf', - parameters: { - layout: 'organic', - endpoint: 'Organic-ppv-zl6ifm7', - }, - }, - { - slug: 'shoplyftermylf', - name: 'Shoplyfter MYLF', - url: 'https://www.shoplyftermylf.com', - tags: ['milf'], - parent: 'mylf', - parameters: { - layout: 'organic', - endpoint: 'MSL-organic-ws9h564all', - }, - }, { slug: 'stayhomemilf', name: 'Stay Home MILF', @@ -14130,20 +14077,6 @@ const sites = [ parameters: { id: 'messy-jessy' }, parent: 'teamskeet', }, - { - slug: 'pervdriver', - name: 'Perv Driver', - url: 'https://www.teamskeet.com/series/pervdriver', - parameters: { id: 'pervdriver' }, - parent: 'teamskeet', - }, - { - slug: 'freakyfembots', - name: 'Freaky Fembots', - url: 'https://www.teamskeet.com/series/freakyfembots', - parameters: { id: 'freakyfembots' }, - parent: 'teamskeet', - }, { slug: 'teamskeetclassics', name: 'Team Skeet Classics', @@ -14700,18 +14633,6 @@ const sites = [ }, parent: 'teamskeet', }, - { - slug: 'daughterswap', - name: 'Daughter Swap', - alias: ['dsw'], - url: 'https://www.daughterswap.com', - tags: ['family'], - parameters: { - layout: 'organic', - endpoint: 'DSW-organic-dfangeym88', - }, - parent: 'teamskeet', - }, { slug: 'fostertapes', name: 'Foster Tapes', @@ -14723,16 +14644,6 @@ const sites = [ }, parent: 'teamskeet', }, - { - slug: 'freeusefantasy', - name: 'Freeuse Fantasy', - url: 'https://www.freeusefantasy.com', - parameters: { - layout: 'organic', - endpoint: 'organic-fuf-eiBei5In', - }, - parent: 'teamskeet', - }, { slug: 'hijabhookup', name: 'Hijab Hookup', @@ -14767,17 +14678,6 @@ const sites = [ }, parent: 'teamskeet', }, - { - slug: 'pervdoctor', - name: 'Perv Doctor', - url: 'https://www.pervdoctor.com', - tags: ['doctor'], - parameters: { - layout: 'organic', - endpoint: 'organic-doc-utei5Mai', - }, - parent: 'teamskeet', - }, { slug: 'pervmom', name: 'Perv Mom', @@ -14789,36 +14689,6 @@ const sites = [ }, parent: 'teamskeet', }, - { - slug: 'pervtherapy', - name: 'Perv Therapy', - url: 'https://www.pervtherapy.com', - parameters: { - layout: 'organic', - endpoint: 'organic-pvt-fePaiz9a', - }, - parent: 'teamskeet', - }, - { - slug: 'shoplyfter', - name: 'Shoplyfter', - url: 'https://www.shoplyfter.com', - parameters: { - layout: 'organic', - endpoint: 'SHL-organic-driobt7t0f', - }, - parent: 'teamskeet', - }, - { - slug: 'sisswap', - name: 'Sis Swap', - url: 'https://www.sisswap.com', - parameters: { - layout: 'organic', - endpoint: 'organic-swp-Jo4daep7', - }, - parent: 'teamskeet', - }, { slug: 'thickumz', name: 'Thickumz', @@ -14850,6 +14720,203 @@ const sites = [ parent: 'teamskeet', }, */ + // PERVZ + { + slug: 'pervzfeatures', + name: 'Pervz Features', + url: 'https://www.pervz.com/series/pervz-features', + parent: 'pervz', + parameters: { + id: 'pervz-features', + endpoint: 'pervbundle', + }, + }, + { + slug: 'pervzsingles', + name: 'Pervz Singles', + url: 'https://www.pervz.com/series/pervz-singles', + parent: 'pervz', + parameters: { + id: 'pervz-singles', + endpoint: 'pervbundle', + }, + }, + { + slug: 'shoplyfter', + name: 'Shoplyfter', + url: 'https://www.shoplyfter.com', + parent: 'pervz', + parameters: { + layout: 'search', + endpoint: 'ts_shl', + }, + }, + { + slug: 'shoplyftermylf', + name: 'Shoplyfter MYLF', + url: 'https://www.shoplyftermylf.com', + tags: ['milf'], + parent: 'pervz', + parameters: { + layout: 'search', + endpoint: 'mylf_msl', + }, + }, + { + slug: 'milftaxi', + name: 'MILF Taxi', + url: 'https://www.pervz.com/series/milf-taxi-ts', + parent: 'pervz', + parameters: { + id: 'milf-taxi-ts', + endpoint: 'pervbundle', + }, + }, + { + slug: 'pervdoctor', + name: 'Perv Doctor', + url: 'https://www.pervz.com/series/pervdoctor', + tags: ['doctor'], + parent: 'pervz', + parameters: { + id: 'pervdoctor', + endpoint: 'pervbundle', + }, + }, + { + slug: 'pervmassage', + name: 'Perv Massage', + url: 'https://www.pervz.com/series/pervmassage', + tags: ['massage'], + parent: 'pervz', + parameters: { + id: 'pervmassage', + endpoint: 'pervbundle', + }, + }, + { + slug: 'pervprincipal', + name: 'Perv Principal', + url: 'https://www.pervprincipal.com', + parent: 'pervz', + parameters: { + layout: 'search', + endpoint: 'mylf_ppv', + }, + }, + { + slug: 'pervtherapy', + name: 'Perv Therapy', + url: 'https://www.pervtherapy.com', + parent: 'pervz', + parameters: { + layout: 'search', + endpoint: 'ts_pvt', + }, + }, + { + slug: 'pervdriver', + name: 'Perv Driver', + url: 'https://www.pervz.com/series/pervdriver', + parent: 'pervz', + parameters: { + id: 'pervdriver', + endpoint: 'pervbundle', + }, + }, + // SWAPPZ + { + slug: 'swappzfeatures', + name: 'Swappz Features', + url: 'https://www.swappz.com/series/swappz-features', + tags: ['family'], + parent: 'swappz', + parameters: { + id: 'swappz-features', + }, + }, + { + slug: 'swappzsingles', + name: 'Swappz Singles', + url: 'https://www.swappz.com/series/swappz-singles', + tags: ['family'], + parent: 'swappz', + parameters: { + id: 'swappz-singles', + }, + }, + { + slug: 'daughterswap', + name: 'Daughter Swap', + url: 'https://www.swappz.com/series/daughterswap', + alias: ['dsw'], + tags: ['family'], + parent: 'swappz', + parameters: { + id: 'daughterswap', + }, + }, + { + slug: 'sisswap', + name: 'Sis Swap', + url: 'https://www.swappz.com/series/sis-swap', + tags: ['family'], + parent: 'swappz', + parameters: { + id: 'sis-swap', + }, + }, + { + slug: 'momswap', + name: 'Mom Swap', + url: 'https://www.swappz.com/series/mom-swap', + tags: ['family', 'milf'], + parent: 'swappz', + parameters: { + id: 'mom-swap', + }, + }, + // FREE USE + { + slug: 'freeusemilf', + name: 'Freeuse MILF', + url: 'https://www.freeuse.com/series/freeuse-milf', + tags: ['free-use', 'milf'], + parent: 'freeuse', + parameters: { + id: 'freeuse-milf', + }, + }, + { + slug: 'freeusefantasy', + name: 'Freeuse Fantasy', + url: 'https://www.freeuse.com/series/freeuse-fantasy', + parent: 'freeuse', + tags: ['free-use'], + parameters: { + id: 'freeuse-fantasy', + }, + }, + { + slug: 'freakyfembots', + name: 'Freaky Fembots', + url: 'https://www.freeuse.com/series/freakyfembots', + parent: 'freeuse', + tags: ['free-use'], + parameters: { + id: 'freakyfembots', + }, + }, + { + slug: 'usepov', + name: 'UsePOV', + url: 'https://www.freeuse.com/series/use-pov', + parent: 'freeuse', + tags: ['free-use', 'pov'], + parameters: { + id: 'use-pov', + }, + }, // TEEN CORE CLUB { name: 'Analyzed Girls', diff --git a/src/scrapers/actors.js b/src/scrapers/actors.js index 0131646f..1ad73b1d 100644 --- a/src/scrapers/actors.js +++ b/src/scrapers/actors.js @@ -193,6 +193,13 @@ module.exports = { // first anal quest doubleviewcasting: firstanalquest, firstanalquest, + // paper street media / team skeet + teamskeet, + sayuncle: teamskeet, + mylf: teamskeet, + pervz: teamskeet, + swappz: teamskeet, + freeuse: teamskeet, // etc '18vr': badoink, theflourishxxx: theflourish, @@ -228,7 +235,6 @@ module.exports = { littlecapricedreams, mariskax, missax, - mylf: teamskeet, nebraskacoeds: elevatedx, pascalssubsluts, pervcity, @@ -239,7 +245,6 @@ module.exports = { private: privateNetwork, realvr: badoink, rickysroom, - sayuncle: teamskeet, score, sexlikereal, spermmania: snowvalley, @@ -253,7 +258,6 @@ module.exports = { uralesbian: snowvalley, rawattack: spizoo, spizoo, - teamskeet, // teencoreclub, teenmegaworld, testedefudelidade, diff --git a/src/scrapers/releases.js b/src/scrapers/releases.js index 3a5b538d..1365a491 100644 --- a/src/scrapers/releases.js +++ b/src/scrapers/releases.js @@ -89,9 +89,18 @@ module.exports = { brazzers: aylo, gaywire: aylo, sexyhub: aylo, - // daringsex, - // arch angel + // paper street media / team skeet + teamskeet, + sayuncle: teamskeet, + mylf: teamskeet, + pervz: teamskeet, + swappz: teamskeet, + freeuse: teamskeet, // etc + analvids: pornbox, + kellymadison, + kink, + '8kmembers': kellymadison, amateurallure: julesjordan, americanpornstar, amateureuro: porndoe, @@ -133,19 +142,13 @@ module.exports = { jesseloadsmonsterfacials, julesjordan, karups, - kellymadison, - '8kmembers': kellymadison, - kink, - // kinkvr: badoink, - // analvids, - analvids: pornbox, + // etc. littlecapricedreams, loveherfilms, mamacitaz: porndoe, mariskax, mikeadriano, missax, - mylf: teamskeet, naughtyamerica, nebraskacoeds: elevatedx, newsensations, @@ -161,7 +164,6 @@ module.exports = { private: privateNetwork, radical, rickysroom, - sayuncle: teamskeet, score, sexlikereal, snowvalley, @@ -169,7 +171,6 @@ module.exports = { swallowsalon: julesjordan, teencoreclub, teenmegaworld, - teamskeet, testedefudelidade, theflourish, tokyohot, diff --git a/src/scrapers/teamskeet.js b/src/scrapers/teamskeet.js index 98a06554..e3cf1c26 100755 --- a/src/scrapers/teamskeet.js +++ b/src/scrapers/teamskeet.js @@ -22,7 +22,26 @@ function getChannelSlug(channelName, entity) { return channel?.slug || null; } -function scrapeScene(scene, channel, parameters) { +async function fetchTrailerUrl(videoId, entity) { + const res = await unprint.get(`https://sstg.psmcode.com/?videoId=${videoId}`, { + headers: { + Origin: entity.origin, + 'sec-fetch-site': 'cross-site', + }, + }); + + if (res.ok) { + const token = res.data.token; + + if (token) { + return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`; + } + } + + return null; +} + +async function scrapeScene(scene, channel, parameters, includeTrailers) { const release = {}; release.entryId = scene.id; @@ -35,7 +54,7 @@ function scrapeScene(scene, channel, parameters) { // release.actors = scene.models?.map((model) => model.modelName) || []; release.actors = scene.models?.map((model) => ({ name: model.modelName || model.name || model.title, - avatar: parameters.avatars && `${parameters.avatars}/${slugify(model.modelName || model.name || model.title, '_')}.jpg`, + avatar: model.img || (parameters.avatars && `${parameters.avatars}/${slugify(model.modelName || model.name || model.title, '_')}.jpg`), url: `${channel.url}/models/${model.modelId || model.id}`, })); @@ -51,8 +70,9 @@ function scrapeScene(scene, channel, parameters) { release.teaser = scene.videoTrailer; - if (scene.video) { - release.trailer = `https://cloudflarestream.com/${scene.video}/manifest/video.mpd?parentOrigin=${encodeURIComponent(channel.url)}`; + if (scene.video && includeTrailers) { + // release.trailer = `https://cloudflarestream.com/${scene.video}/manifest/video.mpd?parentOrigin=${encodeURIComponent(channel.url)}`; + release.trailer = await fetchTrailerUrl(scene.video, channel); } release.tags = scene.tags; @@ -66,7 +86,7 @@ function scrapeScene(scene, channel, parameters) { } function scrapeAll(scenes, channel, parameters) { - return scenes.map((scene) => scrapeScene(scene, channel, parameters)); + return Promise.all(scenes.map(async (scene) => scrapeScene(scene, channel, parameters))); } async function fetchLatest(channel, page = 1, { parameters }) { @@ -99,7 +119,11 @@ async function fetchLatestOrganic(channel, page, context) { } async function fetchLatestSearch(channel, page = 1, { parameters }) { - const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:%22${parameters.id}%22%20AND%20type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); + const url = parameters.id + ? `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:"${parameters.id}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}` + : `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`; + + const res = await http.get(url); if (res.ok) { return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); @@ -108,8 +132,8 @@ async function fetchLatestSearch(channel, page = 1, { parameters }) { return res.status; } -async function fetchScene(url, channel, baseScene, { parameters }) { - if (parameters.layout !== 'organic' && baseScene?.entryId) { +async function fetchScene(url, channel, baseScene, { parameters, includeTrailers }) { + if (parameters.layout !== 'organic' && baseScene?.entryId && !includeTrailers) { // overview and deep data is the same in elastic API, don't hit server unnecessarily return baseScene; } @@ -134,7 +158,7 @@ async function fetchScene(url, channel, baseScene, { parameters }) { const video = videos?.[sceneSlug] || Object.values(videos)[0]; if (video) { - return scrapeScene(video, channel, parameters); + return scrapeScene(video, channel, parameters, includeTrailers); } return null; @@ -143,7 +167,7 @@ async function fetchScene(url, channel, baseScene, { parameters }) { return res.status; } -function scrapeProfile(actor, entity, parameters) { +async function scrapeProfile(actor, entity, parameters) { const profile = {}; profile.url = `${entity.url}/models/${actor.id}`; @@ -203,7 +227,7 @@ function scrapeProfile(actor, entity, parameters) { profile.avatar = actor.img; profile.banner = actor.cover; - profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity, parameters)); + profile.scenes = await Promise.all(actor.movies?.map(async (scene) => scrapeScene(scene, entity, parameters))); return profile; }