DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Fixed Teen Mega World scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2023-11-07 03:06:39 +01:00
parent 8f81f71802
commit e20641e101
102 changed files with 86 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
src/scrapers/teenmegaworld.js
View File

@@ -1,6 +1,7 @@
'use strict';
const qu = require('../utils/q');
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, channel) {
@@ -8,104 +9,110 @@ function scrapeAll(scenes, channel) {
const release = {};
const network = channel.type === 'network' ? channel : channel.parent;
release.url = query.url('a.title');
release.url = query.url('.thumb__title-link');
release.entryId = new URL(release.url).pathname.match(/\/trailers\/(.+).html/)?.[1].toLowerCase();
release.title = query.cnt('.title');
release.title = query.content('.thumb__title-link');
release.date = query.date('time', unprint.dateConstants.DATETIME_LOCAL_SECONDS, { attribute: 'datetime' }) || query.date('time', 'MMMM D, YYYY');
release.date = query.date('time', 'MMMM D, YYYY');
release.actors = query.all('.actors a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
release.actors = query.all('.actor__link').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null),
}));
release.poster = query.img('.thumb__picture img');
[release.poster, ...release.photos] = Object
.entries(query.el('.images img').dataset)
.filter(([key]) => /src/.test(key))
.map(([, value]) => qu.prefixUrl(value, network.url));
.entries(query.el('.thumb__image').dataset)
.filter(([key]) => /^src/.test(key))
.map(([, value]) => [value.replace('-1x', '-2x'), value].map((path) => unprint.prefixUrl(path, network.url)));
release.likes = query.number('.rating-on-thumb');
release.teaser = query.video('.thumb__video', { origin: network.url });
const siteId = query.url('.site a', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
release.likes = query.number('.thumb__rating-value');
if (siteId) {
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
const channelSlug = slugify(query.content('.thumb__detail__site-link'), '');
if (channelSlug) {
release.channel = network.children.find((child) => child.slug === channelSlug)?.slug;
}
return release;
});
}
function scrapeScene({ query }, url, channel) {
function scrapeScene({ query }, context) {
const release = {};
const network = channel.type === 'network' ? channel : channel.parent;
release.entryId = new URL(url).pathname.match(/\/trailers\/(.+).html/)[1].toLowerCase();
release.entryId = new URL(context.url).pathname.match(/\/trailers\/(.+).html/)[1].toLowerCase();
release.title = query.cnt('.title-line h1');
release.description = query.cnt('.shadow .text');
release.title = query.content('.video-title');
release.description = query.content('.video-description-text');
release.date = query.date('.title-line .date', 'MMMM D, YYYY');
release.duration = query.number('.dur') * 60;
release.date = query.date('.video-info-date', 'MMMM D, YYYY');
release.duration = query.number('.video-info-time') * 60;
release.actors = query.all('.site a[href*="/models"]').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
release.actors = query.all('.video-actor-link').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null),
}));
release.tags = query.cnts('.tag-list a');
release.tags = query.contents('.video-tag-link');
const poster = query.img('.video-cover__image');
if (!poster.includes('error')) {
release.poster = poster;
}
release.poster = query.poster();
release.trailer = query.video();
release.likes = query.number('.rating-data .value');
const siteId = query.url('.site a[href*="site[]"]', 'href', { origin: network.url, object: true })?.searchParams.get('site[]');
if (siteId) {
release.channel = network.children.find((child) => child.parameters.siteId.toString() === siteId)?.slug;
}
release.likes = query.number('.video-actions-button [data-count]');
release.channel = slugify(query.content('.video-site-link'), '');
return release;
}
function scrapeProfile({ query, el }, url, entity) {
function scrapeProfile({ query }, url, entity) {
const profile = {};
const bio = query.all('.stats li').reduce((acc, bioEl) => ({
const bio = query.all('.model-profile-information-element').reduce((acc, bioEl) => ({
...acc,
[slugify(query.cnt(bioEl, '.title'))]: query.cnt(bioEl, '.value'),
[slugify(unprint.query.content(bioEl, '.model-profile-information-key'))]: unprint.query.text(bioEl),
}), {});
profile.url = url;
profile.description = query.cnt('.bio .text');
profile.description = query.content('.model-profile-about');
profile.hairColor = bio.hair;
profile.eyes = bio.eyes;
profile.avatar = query.img('.model-info .photo img', 'data-src', { origin: entity.url });
profile.scenes = scrapeAll(qu.initAll(el, '.video-list .video[data-type="vids"]'), entity);
profile.avatar = query.img('.model-profile-image-picture source', { origin: entity.url, attribute: 'srcset' }) || query.img('.model-profile-image-picture img', { origin: entity.url });
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-list .thumb')), entity);
console.log(bio);
console.log(profile);
return profile;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.type === 'network' ? channel.url : channel.parent.url}/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
const res = await qu.getAll(url, '.video-list .video');
const res = await unprint.get(url, { selectAll: '.video-list .thumb' });
if (res.ok) {
return scrapeAll(res.items, channel);
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchProfile({ name: actorName }, entity, include) {
const url = `${entity.url}/models/${slugify(actorName)}.html`;
const res = await qu.get(url);
async function fetchProfile(actor, entity, include) {
const url = actor.url || `${entity.url}/models/${slugify(actor.name)}.html`;
const res = await unprint.get(url);
if (res.ok) {
return scrapeProfile(res.item, url, entity, include);
return scrapeProfile(res.context, url, entity, include);
}
return res.status;
@@ -114,5 +121,8 @@ async function fetchProfile({ name: actorName }, entity, include) {
module.exports = {
fetchLatest,
fetchProfile,
scrapeScene,
scrapeScene: {
scraper: scrapeScene,
unprint: true,
},
};