Fixed media module trying to fetch invalid source URLs. Added Accidental Gangbang to Adult Time.

This commit is contained in:
DebaucheryLibrarian 2022-09-27 20:09:46 +02:00
parent 3db8b80164
commit 0fc37e46d2
29 changed files with 4857 additions and 2451 deletions

View File

@ -431,6 +431,22 @@ const releasesFragment = `
} }
} }
} }
or: [
{
entity: {
slug: {
notEqualTo: "analvids"
}
}
}
{
studio: {
slug: {
in: ["giorgiograndi"]
}
}
}
],
}, },
first: $limit, first: $limit,
offset: $offset, offset: $offset,

View File

@ -297,6 +297,11 @@ module.exports = {
enable: true, enable: true,
auto: true, // try bypass when CF challenge is detected auto: true, // try bypass when CF challenge is detected
path: 'http://localhost:8191/v1', path: 'http://localhost:8191/v1',
sharedHostnames: [ // these can run in the same browser session
'store2.psmcdn.net', // Team Skeet API
'www.kink.com',
],
independentHostnames: [], // these must run in their own browser session
}, },
}, },
limits: { limits: {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.0 KiB

After

Width:  |  Height:  |  Size: 7.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.3 KiB

After

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.1 KiB

After

Width:  |  Height:  |  Size: 5.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

View File

@ -363,6 +363,19 @@ const sites = [
}, },
}, },
// ADULT TIME // ADULT TIME
{
name: 'Accidental Gangbang',
slug: 'accidentalgangbang',
url: 'https://accidentalgangbang.com',
parent: 'adulttime',
tags: ['gangbang'],
parameters: {
referer: 'https://freetour.adulttime.com/en/join',
deep: 'https://21sextury.com/en/video',
scene: false,
includePhotos: false,
},
},
{ {
name: 'ASMR Fantasy', name: 'ASMR Fantasy',
slug: 'asmrfantasy', slug: 'asmrfantasy',

File diff suppressed because it is too large Load Diff

View File

@ -199,6 +199,8 @@ async function init() {
logger.error(error); logger.error(error);
} }
await http.destroyBypassSessions();
knex.destroy(); knex.destroy();
done = true; done = true;
} }

View File

@ -96,15 +96,39 @@ function itemsByKey(items, key) {
return items.reduce((acc, item) => ({ ...acc, [item[key]]: item }), {}); return items.reduce((acc, item) => ({ ...acc, [item[key]]: item }), {});
} }
function isValidUrl(url) {
try {
const urlObject = new URL(url);
return !!urlObject;
} catch (error) {
return false;
}
}
function toBaseSource(rawSource) { function toBaseSource(rawSource) {
if (rawSource && (rawSource.src || (rawSource.extract && rawSource.url) || rawSource.stream)) { if (rawSource && (rawSource.src || (rawSource.extract && rawSource.url) || rawSource.stream)) {
const baseSource = {}; const baseSource = {};
if (rawSource.src) baseSource.src = rawSource.src; if (rawSource.src) {
if (!isValidUrl(rawSource.src)) {
return null;
}
baseSource.src = rawSource.src;
}
if (rawSource.quality) baseSource.quality = rawSource.quality; if (rawSource.quality) baseSource.quality = rawSource.quality;
if (rawSource.type) baseSource.type = rawSource.type; if (rawSource.type) baseSource.type = rawSource.type;
if (rawSource.url) baseSource.url = rawSource.url; if (rawSource.url) {
if (!isValidUrl(rawSource.url)) {
return null;
}
baseSource.url = rawSource.url;
}
if (rawSource.extract) baseSource.extract = rawSource.extract; if (rawSource.extract) baseSource.extract = rawSource.extract;
if (rawSource.expectType) baseSource.expectType = rawSource.expectType; if (rawSource.expectType) baseSource.expectType = rawSource.expectType;
@ -135,6 +159,10 @@ function toBaseSource(rawSource) {
} }
if (typeof rawSource === 'string') { if (typeof rawSource === 'string') {
if (!isValidUrl(rawSource)) {
return null;
}
return { return {
src: rawSource, src: rawSource,
}; };

View File

@ -150,7 +150,7 @@ async function getFullPhotos(entryId, site) {
}, },
}); });
if (res.ok) { if (res.ok && typeof res.body === 'object') { // gives 200 OK even when redirected to signup page
return Object.values(res.body); return Object.values(res.body);
} }
@ -442,7 +442,7 @@ async function scrapeReleaseApi(data, site, options) {
]; ];
} }
if (data.photoset_id && options.includePhotos) { if (data.photoset_id && options.includePhotos && options.parameters?.includePhotos !== false) {
release.photos = await getPhotosApi(data.photoset_id, site, options.parameters); release.photos = await getPhotosApi(data.photoset_id, site, options.parameters);
} }

View File

@ -132,13 +132,9 @@ async function scrapeProfile({ query }, actorUrl, include) {
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', { // const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', { // const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
Host: 'www.kink.com', const res = await qu.getAll(`https://www.kink.com/search?type=shoots&channelIds=${site.slug}&sort=published&page=${page}`, '.shoot-list .shoot', {
'User-Agent': 'HTTPie/2.6.0', ct: 2,
'Accept-Encoding': 'gzip, deflate, br',
Accept: '*/*',
Connection: 'keep-alive',
}, { }, {
includeDefaultHeaders: false, includeDefaultHeaders: false,
followRedirects: false, followRedirects: false,

View File

@ -119,11 +119,7 @@ function scrapeProfile(actor, entity) {
} }
async function fetchLatest(channel, page = 1, { parameters }) { async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, { const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
bypassCloudflare: true,
});
console.log(res.status);
if (res.ok) { if (res.ok) {
return scrapeAll(res.body.hits.hits, channel); return scrapeAll(res.body.hits.hits, channel);

File diff suppressed because it is too large Load Diff

View File

@ -26,6 +26,8 @@ const limiters = {
}), }),
}; };
const bypassSessions = new Map();
Promise.config({ Promise.config({
cancellation: true, cancellation: true,
}); });
@ -56,6 +58,32 @@ function useProxy(url) {
return config.proxy.hostnames.includes(hostname); return config.proxy.hostnames.includes(hostname);
} }
function useCloudflareBypass(url, options) {
if (!config.bypass.cloudflare.enable) {
return null;
}
const { hostname } = new URL(url);
if (options.bypassCloudflare === 'shared') {
return 'shared';
}
if (options.bypassCloudflare === 'independent') {
return hostname;
}
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
return 'shared';
}
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
return hostname;
}
return null;
}
function getLimiterValue(prop, options, hostname) { function getLimiterValue(prop, options, hostname) {
if (argv[prop] !== undefined) { if (argv[prop] !== undefined) {
return argv[prop]; return argv[prop];
@ -106,11 +134,72 @@ function extractJson(solution) {
return solution.response; return solution.response;
} }
async function bypassCloudflareRequest(url, method, body, options) { async function getBypassSession(url, hostname) {
if (bypassSessions.has(hostname)) {
return bypassSessions.get(hostname);
}
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.create',
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
});
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
}
bypassSessions.set(hostname, sessionRes.body.session);
return sessionRes.body.session;
}
async function destroyBypassSession(sessionId) {
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.destroy',
session: sessionId,
}, {
encodeJSON: true,
}));
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
bypassSessions.delete(sessionId);
logger.verbose(`Destroyed bypass session ${sessionId}`);
return true;
}
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
return false;
}
async function destroyBypassSessions() {
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.list',
}, {
encodeJSON: true,
}));
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
}
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
}
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL // the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, { const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: `request.${method}`, cmd: `request.${method}`,
url, url,
session: sessionId,
maxTimeout: options.timeout, maxTimeout: options.timeout,
proxy: useProxy(url) ? { proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`, url: `${config.proxy.host}:${config.proxy.port}`,
@ -120,6 +209,12 @@ async function bypassCloudflareRequest(url, method, body, options) {
})); }));
if (!res.statusCode === 200 || res.body?.status !== 'ok') { if (!res.statusCode === 200 || res.body?.status !== 'ok') {
if (/session closed/i.test(res.body?.message) && attempts < 3) {
await destroyBypassSession(sessionId);
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
}
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`); throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
} }
@ -141,7 +236,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
}; };
const withProxy = useProxy(url); const withProxy = useProxy(url);
const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable; const withCloudflareBypass = useCloudflareBypass(url, options);
if (withProxy) { if (withProxy) {
options.agent = proxyAgent; options.agent = proxyAgent;
@ -150,7 +245,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`); logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
if (withCloudflareBypass) { if (withCloudflareBypass) {
return bypassCloudflareRequest(url, method, body, options); return bypassCloudflareRequest(url, method, body, withCloudflareBypass, options);
} }
const res = await (body const res = await (body
@ -292,4 +387,5 @@ module.exports = {
cookieJar: getCookieJar, cookieJar: getCookieJar,
getSession, getSession,
getCookieJar, getCookieJar,
destroyBypassSessions,
}; };