DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 36 Pull Requests 1 Releases Wiki Activity

Using network page for Dogfart sites without complete index.

This commit is contained in:
DebaucheryLibrarian 2022-04-07 23:22:40 +02:00
parent 7c46bdd495
commit cc63cc652a
4 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
seeds/02_sites.js
View File

@ -2743,6 +2743,9 @@ const sites = [
alias: ['wfbg'],
url: 'https://www.wefuckblackgirls.com',
parent: 'dogfartnetwork',
parameters: {
latest: 'https://www.dogfartnetwork.com/tour/sites/WeFuckBlackGirls',
},
},
{
slug: 'watchingmymomgoblack',
@ -2767,6 +2770,9 @@ const sites = [
name: 'Interracial Pickups',
url: 'https://www.interracialpickups.com',
parent: 'dogfartnetwork',
parameters: {
latest: 'https://www.dogfartnetwork.com/tour/sites/InterracialPickups',
},
},
{
slug: 'watchingmydaughtergoblack',
@ -2850,6 +2856,9 @@ const sites = [
url: 'https://www.blacksonboys.com',
parent: 'dogfartnetwork',
tags: ['gay'],
parameters: {
latest: 'https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys',
},
},
{
slug: 'gloryholesandhandjobs',
@ -2857,6 +2866,9 @@ const sites = [
url: 'https://www.gloryholesandhandjobs.com',
parent: 'dogfartnetwork',
tags: ['gay'],
parameters: {
latest: 'https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs',
},
},
// DORCEL
{

2
src/scrapers/dogfart.js
View File

@ -52,7 +52,7 @@ function scrapeLatest(scenes, site, filter = true) {
release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) {
// different dogfart site
return { ...acc, unextracted: [...acc.unextracted, release] };
}

10
src/tools/dogfart.js
View File

@ -40,19 +40,19 @@ async function compare() {
const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
await fs.writeFile('./dogfart-new', getLinks.join('\n'));
await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
console.log(getLinks);
}
*/
async function scrapeMembers() {
const titlesByPage = await Promise.map(Array.from({ length: 1 }), async (value, index) => {
const res = await qu.get(`https://sbj1e2bdv33.dogfartnetwork.com/members/blacksonboys/index.php?page=${index + 1}`);
const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
if (res.ok) {
return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
url: `https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
actors: query.contents('a[href*="model.php"]'),
trailer: query.urls('.trailer-link a').map((url) => ({
src: url,
@ -68,7 +68,7 @@ async function scrapeMembers() {
console.log(urls);
await fs.writeFile('./dogfart-blacksonboys', Array.from(new Set(urls)).join('\n'));
await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}

2
src/updates.js
View File

@ -75,6 +75,8 @@ async function filterUniqueReleases(releases) {
function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
console.log(uniquePageReleases.length, unextracted.length);
if (uniquePageReleases.length + unextracted.length === 0) {
// page is empty, or only contains scenes from previous page
return false;