DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Scraping Dogfart scenes from native sites.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2022-04-03 23:00:05 +02:00
parent 08df432665
commit 5cbf122d6f
2 changed files with 41 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/scrapers/dogfart.js
View File

@@ -10,7 +10,7 @@ async function getPhotos(albumUrl) {
return [];
}
const lastPhotoPage = res.item.query.urls('.preview-image-container a').at(-1);
const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1);
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
@@ -31,19 +31,19 @@ function scrapeLatest(scenes, site, filter = true) {
const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name');
release.url = query.url('.thumbnail', 'href', { origin: site.type === 'network' ? site.url : site.parent.url });
release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url });
release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`;
release.title = query.cnt('.scene-title');
release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
// release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com
// release.poster = `https:${element.querySelector('img').src}`;
release.poster = query.img();
release.teaser = query.el('.thumbnail', 'data-preview_clip_url');
release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url');
release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
if (filter && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
// different dogfart site
return { ...acc, unextracted: [...acc.unextracted, release] };
}
@@ -59,26 +59,26 @@ async function scrapeScene({ query }, url, channel, baseScene, parameters) {
const release = {};
const { origin, pathname } = new URL(url);
release.channel = query.cnt('.site-name').split('.')[0].toLowerCase();
release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase();
release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`;
release.title = query.cnt('.description-title');
release.actors = query.all('.more-scenes a').map((actorEl) => ({
release.title = query.cnt('.description-title') || query.text('.scene-title');
release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
release.description = query.meta('meta[itemprop="description"]') || qu.cnt('.description').replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim();
release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description').replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim();
release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content');
release.duration = query.duration('.extra-info p:nth-child(2)');
release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container');
release.tags = query.cnts('.scene-details .categories a');
release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/);
const trailer = query.video('.html5-video', 'data-trailer');
const lastPhotosUrl = query.urls('.pagination a').at(-1);
release.poster = query.poster('.html5-video', 'data-poster');
release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image');
if (trailer && !trailer?.includes('join')) {
release.trailer = trailer;
@@ -88,16 +88,17 @@ async function scrapeScene({ query }, url, channel, baseScene, parameters) {
release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url);
}
release.stars = Number(((query.number('span[itemprop="average"]') || query.number('span[itemprop="ratingValue"]')) / 2).toFixed(2));
release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2));
return release;
}
async function fetchLatest(site, page = 1) {
const res = await qu.getAll(`https://dogfartnetwork.com/tour/scenes/?p=${page}`, '.recent-updates');
async function fetchLatest(channel, page = 1) {
// const res = await qu.getAll(`https://dogfartnetwork.com/tour/scenes/?p=${page}`, '.recent-updates');
const res = await qu.getAll(`${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container');
if (res.ok) {
return scrapeLatest(res.items, site);
return scrapeLatest(res.items, channel);
}
return res.status;