diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 6fc3c4f5..9e96b12b 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2743,6 +2743,9 @@ const sites = [ alias: ['wfbg'], url: 'https://www.wefuckblackgirls.com', parent: 'dogfartnetwork', + parameters: { + latest: 'https://www.dogfartnetwork.com/tour/sites/WeFuckBlackGirls', + }, }, { slug: 'watchingmymomgoblack', @@ -2767,6 +2770,9 @@ const sites = [ name: 'Interracial Pickups', url: 'https://www.interracialpickups.com', parent: 'dogfartnetwork', + parameters: { + latest: 'https://www.dogfartnetwork.com/tour/sites/InterracialPickups', + }, }, { slug: 'watchingmydaughtergoblack', @@ -2850,6 +2856,9 @@ const sites = [ url: 'https://www.blacksonboys.com', parent: 'dogfartnetwork', tags: ['gay'], + parameters: { + latest: 'https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys', + }, }, { slug: 'gloryholesandhandjobs', @@ -2857,6 +2866,9 @@ const sites = [ url: 'https://www.gloryholesandhandjobs.com', parent: 'dogfartnetwork', tags: ['gay'], + parameters: { + latest: 'https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs', + }, }, // DORCEL { diff --git a/src/scrapers/dogfart.js b/src/scrapers/dogfart.js index a89aa7e4..7a11709f 100644 --- a/src/scrapers/dogfart.js +++ b/src/scrapers/dogfart.js @@ -52,7 +52,7 @@ function scrapeLatest(scenes, site, filter = true) { release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase(); - if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) { + if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) { // different dogfart site return { ...acc, unextracted: [...acc.unextracted, release] }; } diff --git a/src/tools/dogfart.js b/src/tools/dogfart.js index 1b15d963..c91ad4b3 100644 --- a/src/tools/dogfart.js +++ b/src/tools/dogfart.js @@ -40,19 +40,19 @@ async function compare() { const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`); - await fs.writeFile('./dogfart-new', getLinks.join('\n')); + await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n')); console.log(getLinks); } */ async function scrapeMembers() { - const titlesByPage = await Promise.map(Array.from({ length: 1 }), async (value, index) => { - const res = await qu.get(`https://sbj1e2bdv33.dogfartnetwork.com/members/blacksonboys/index.php?page=${index + 1}`); + const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => { + const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`); if (res.ok) { return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({ - url: `https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`, + url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`, actors: query.contents('a[href*="model.php"]'), trailer: query.urls('.trailer-link a').map((url) => ({ src: url, @@ -68,7 +68,7 @@ async function scrapeMembers() { console.log(urls); - await fs.writeFile('./dogfart-blacksonboys', Array.from(new Set(urls)).join('\n')); + await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n')); console.log(`Saved ${urls.length} URLs to file`); } diff --git a/src/updates.js b/src/updates.js index 7540b47f..3f88d4ec 100644 --- a/src/updates.js +++ b/src/updates.js @@ -75,6 +75,8 @@ async function filterUniqueReleases(releases) { function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) { const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases); + console.log(uniquePageReleases.length, unextracted.length); + if (uniquePageReleases.length + unextracted.length === 0) { // page is empty, or only contains scenes from previous page return false;