Using network page for Dogfart sites without complete index.

seeds/02_sites.js

@@ -2743,6 +2743,9 @@ const sites = [
 		alias: ['wfbg'],
 		url: 'https://www.wefuckblackgirls.com',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/WeFuckBlackGirls',
+		},
 	},
 	{
 		slug: 'watchingmymomgoblack',
@@ -2767,6 +2770,9 @@ const sites = [
 		name: 'Interracial Pickups',
 		url: 'https://www.interracialpickups.com',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/InterracialPickups',
+		},
 	},
 	{
 		slug: 'watchingmydaughtergoblack',
@@ -2850,6 +2856,9 @@ const sites = [
 		url: 'https://www.blacksonboys.com',
 		parent: 'dogfartnetwork',
 		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys',
+		},
 	},
 	{
 		slug: 'gloryholesandhandjobs',
@@ -2857,6 +2866,9 @@ const sites = [
 		url: 'https://www.gloryholesandhandjobs.com',
 		parent: 'dogfartnetwork',
 		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs',
+		},
 	},
 	// DORCEL
 	{

src/scrapers/dogfart.js

@@ -52,7 +52,7 @@ function scrapeLatest(scenes, site, filter = true) {
 
 		release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
 
-		if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
+		if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) {
 			// different dogfart site
 			return { ...acc, unextracted: [...acc.unextracted, release] };
 		}

src/tools/dogfart.js

@@ -40,19 +40,19 @@ async function compare() {
 
 	const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
 
-	await fs.writeFile('./dogfart-new', getLinks.join('\n'));
+	await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
 
 	console.log(getLinks);
 }
 */
 
 async function scrapeMembers() {
-	const titlesByPage = await Promise.map(Array.from({ length: 1 }), async (value, index) => {
+	const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
-		const res = await qu.get(`https://sbj1e2bdv33.dogfartnetwork.com/members/blacksonboys/index.php?page=${index + 1}`);
+		const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
 
 		if (res.ok) {
 			return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
-				url: `https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
+				url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
 				actors: query.contents('a[href*="model.php"]'),
 				trailer: query.urls('.trailer-link a').map((url) => ({
 					src: url,
@@ -68,7 +68,7 @@ async function scrapeMembers() {
 
 	console.log(urls);
 
-	await fs.writeFile('./dogfart-blacksonboys', Array.from(new Set(urls)).join('\n'));
+	await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
 
 	console.log(`Saved ${urls.length} URLs to file`);
 }

src/updates.js

@@ -75,6 +75,8 @@ async function filterUniqueReleases(releases) {
 function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
 	const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
 
+	console.log(uniquePageReleases.length, unextracted.length);
+
 	if (uniquePageReleases.length + unextracted.length === 0) {
 		// page is empty, or only contains scenes from previous page
 		return false;