boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Using network page for Dogfart sites without complete index.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2022-04-07 23:22:40 +02:00
parent 7c46bdd495
commit cc63cc652a
4 changed files with 20 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/scrapers/dogfart.js
View File

@@ -52,7 +52,7 @@ function scrapeLatest(scenes, site, filter = true) {
release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) {
// different dogfart site
return { ...acc, unextracted: [...acc.unextracted, release] };
}

10
src/tools/dogfart.js
View File

@@ -40,19 +40,19 @@ async function compare() {
const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
await fs.writeFile('./dogfart-new', getLinks.join('\n'));
await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
console.log(getLinks);
}
*/
async function scrapeMembers() {
const titlesByPage = await Promise.map(Array.from({ length: 1 }), async (value, index) => {
const res = await qu.get(`https://sbj1e2bdv33.dogfartnetwork.com/members/blacksonboys/index.php?page=${index + 1}`);
const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
if (res.ok) {
return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
url: `https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
actors: query.contents('a[href*="model.php"]'),
trailer: query.urls('.trailer-link a').map((url) => ({
src: url,
@@ -68,7 +68,7 @@ async function scrapeMembers() {
console.log(urls);
await fs.writeFile('./dogfart-blacksonboys', Array.from(new Set(urls)).join('\n'));
await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
console.log(`Saved ${urls.length} URLs to file`);
}

2
src/updates.js
View File

@@ -75,6 +75,8 @@ async function filterUniqueReleases(releases) {
function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
console.log(uniquePageReleases.length, unextracted.length);
if (uniquePageReleases.length + unextracted.length === 0) {
// page is empty, or only contains scenes from previous page
return false;