Refactored media fetch with undici for http/2 support.
This commit is contained in:
DebaucheryLibrarian
@@ -85,7 +85,7 @@ async function scrapeScene({ query: pageQuery, html }, { url, entity, include })
|
||||
}
|
||||
|
||||
if (include.photos && capsUrl) {
|
||||
release.caps = await fetchCaps(capsUrl);
|
||||
release.caps = await fetchCaps(capsUrl, entity);
|
||||
}
|
||||
|
||||
release.trailer = pageQuery.video('#download_select option[value*=".mp4"]', { attribute: 'value' });
|
||||
|
||||
+56
-127
@@ -1,155 +1,84 @@
|
||||
'use strict';
|
||||
|
||||
const moment = require('moment');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
const http = require('../utils/http');
|
||||
const qu = require('../utils/qu');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { prefixUrl } = require('../utils/qu');
|
||||
|
||||
function scrapeAll(scenes, entity) {
|
||||
return scenes.map((scene) => {
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.entryId = scene.id;
|
||||
release.url = `${new URL(entity.url).origin}/video/${scene.id}/${scene.slug}`;
|
||||
release.url = query.url('.title a');
|
||||
release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1];
|
||||
|
||||
if (/bic/i.test(scene.title)) {
|
||||
release.shootId = scene.title.toUpperCase().replace('-', '_');
|
||||
} else {
|
||||
release.title = scene.title;
|
||||
}
|
||||
release.title = query.content('.title a');
|
||||
|
||||
release.description = scene.description;
|
||||
release.date = moment.utc(scene.year, 'YYYY').toDate();
|
||||
release.datePrecision = 'year';
|
||||
release.date = query.date('.date', 'MMM DD, YYYY');
|
||||
release.duration = query.duration('.duration');
|
||||
|
||||
release.actors = scene.actors.map((actor) => ({
|
||||
name: actor.name.trim(),
|
||||
avatar: actor.image || null,
|
||||
})).filter((actor) => actor.name && slugify(actor.name) !== 'amateur-girl');
|
||||
release.actors = query.all('.models a.model').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.duration = scene.duration;
|
||||
release.stars = scene.video_rating_score;
|
||||
release.poster = query.img('img.poster');
|
||||
release.teaser = query.video('.teaser video');
|
||||
|
||||
[release.poster, ...release.photos] = scene.screenshots.map((url) => prefixUrl(url));
|
||||
|
||||
if (scene.is_gay) {
|
||||
release.tags = ['gay'];
|
||||
}
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, url) {
|
||||
const release = {};
|
||||
const { pathname, origin, host } = new URL(url);
|
||||
|
||||
const entryId = pathname.match(/\/video\/(\d+)/)[1];
|
||||
release.entryId = entryId;
|
||||
|
||||
const title = query.meta('name=title');
|
||||
|
||||
if (/bic/i.test(title)) {
|
||||
release.shootId = title.toUpperCase().replace('-', '_');
|
||||
} else {
|
||||
release.title = title;
|
||||
}
|
||||
|
||||
release.date = query.date('.detail-meta li:nth-child(2)', 'YYYY');
|
||||
release.datePrecision = 'year';
|
||||
|
||||
release.description = query.q('.detail-description', true);
|
||||
release.duration = query.dur('.detail-meta li:first-child');
|
||||
|
||||
const actors = [query.q('.detail-hero-title h1', true)?.trim()].filter((name) => name && slugify(name) !== 'amateur-girl');
|
||||
|
||||
if (actors.length > 0) {
|
||||
release.actors = actors;
|
||||
}
|
||||
|
||||
release.poster = query.q('.detail-hero').style['background-image'].match(/url\((.+)\)/)[1];
|
||||
release.photos = query.imgs('.detail-grabs img');
|
||||
|
||||
const streamData = await http.get(`${origin}/video/source/${entryId}`, {
|
||||
headers: {
|
||||
host,
|
||||
referer: url,
|
||||
},
|
||||
}, {
|
||||
interval: 5000,
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
if (streamData.ok && streamData.body.status === 'success') {
|
||||
release.trailer = {
|
||||
stream: streamData.body.link,
|
||||
};
|
||||
} else {
|
||||
logger.warn(`Failed to fetch trailer for ${url}: ${streamData.ok ? streamData.body.status : streamData.status }`);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function scrapeProfile(actor, entity, include) {
|
||||
const profile = {};
|
||||
|
||||
if (actor.image) {
|
||||
profile.avatar = `https://teencoreclub.com${actor.image}`;
|
||||
}
|
||||
|
||||
if (include.releases) {
|
||||
const res = await http.get(`https://teencoreclub.com/browsevideos/api/all?actor=${actor.id}`);
|
||||
|
||||
if (res.ok) {
|
||||
profile.releases = scrapeAll(res.body.data, entity);
|
||||
}
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(entity, page = 1) {
|
||||
// console.log(entity, page);
|
||||
|
||||
if (entity.parameters?.siteId) {
|
||||
const res = await http.get(`https://teencoreclub.com/browsevideos/api/all?resType=latest&page=${page}&label=${entity.parameters.siteId}`);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.body.data, entity);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity) {
|
||||
const { pathname } = new URL(url);
|
||||
const res = await qu.get(`https://teencoreclub.com${pathname}`);
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.scene' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, entity);
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, { entity }, include) {
|
||||
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
|
||||
function scrapeScene({ query }, { url }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1];
|
||||
|
||||
release.title = query.content('h3.title');
|
||||
release.description = query.content('p.description');
|
||||
|
||||
release.date = query.date('.date', 'MMMM D, YYYY');
|
||||
release.duration = query.duration('.duration');
|
||||
|
||||
[release.poster, ...release.photos] = query.imgs('.preview-thumb');
|
||||
release.trailer = query.video('.trailer video');
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
profile.description = query.content('.bio-text');
|
||||
profile.birthPlace = query.content('.birth-place span');
|
||||
|
||||
profile.avatar = query.img('.actor-photo img');
|
||||
|
||||
console.log(profile);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, entity) {
|
||||
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
const actor = res.body.data.find((item) => slugify(item.name) === slugify(actorName));
|
||||
|
||||
if (actor) {
|
||||
return scrapeProfile(actor, entity, include);
|
||||
}
|
||||
|
||||
return null;
|
||||
return scrapeProfile(res.context, entity);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -157,6 +86,6 @@ async function fetchProfile({ name: actorName }, { entity }, include) {
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
scrapeScene,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user