Added Fame Digital. Added actor release scraping to DDF Network. Improved q and Gamma scraper.
This commit is contained in:
@@ -111,18 +111,23 @@ function scrapeAll(html, site, useNetworkUrl) {
|
||||
const scenesElements = $('li[data-itemtype=scene]').toArray();
|
||||
|
||||
return scenesElements.map((element) => {
|
||||
const release = {};
|
||||
|
||||
const sceneLinkElement = $(element).find('.sceneTitle a');
|
||||
|
||||
const url = `${useNetworkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`;
|
||||
const title = sceneLinkElement.attr('title');
|
||||
release.url = `${useNetworkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`;
|
||||
release.title = sceneLinkElement.attr('title');
|
||||
|
||||
const entryId = $(element).attr('data-itemid');
|
||||
release.entryId = $(element).attr('data-itemid');
|
||||
|
||||
const date = moment
|
||||
.utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY')
|
||||
.toDate();
|
||||
const dateEl = $(element).find('.sceneDate').text() || null;
|
||||
if (dateEl) {
|
||||
release.date = moment
|
||||
.utc($(element).find('.sceneDate').text(), ['MM-DD-YYYY', 'YYYY-MM-DD'])
|
||||
.toDate();
|
||||
}
|
||||
|
||||
const actors = $(element).find('.sceneActors a')
|
||||
release.actors = $(element).find('.sceneActors a')
|
||||
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
|
||||
.toArray();
|
||||
|
||||
@@ -130,27 +135,17 @@ function scrapeAll(html, site, useNetworkUrl) {
|
||||
.toArray()
|
||||
.map(value => Number($(value).text()));
|
||||
|
||||
const poster = $(element).find('.imgLink img').attr('data-original');
|
||||
const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
|
||||
release.rating = { likes, dislikes };
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
director: 'Mason',
|
||||
date,
|
||||
poster,
|
||||
trailer: {
|
||||
src: trailer,
|
||||
quality: 224,
|
||||
},
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
},
|
||||
site,
|
||||
const posterEl = $(element).find('.imgLink img');
|
||||
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
|
||||
|
||||
release.teaser = {
|
||||
src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`,
|
||||
quality: 224,
|
||||
};
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -161,33 +156,41 @@ async function scrapeScene(html, url, site) {
|
||||
const json = $('script[type="application/ld+json"]').html();
|
||||
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
|
||||
|
||||
const [data, data2] = JSON.parse(json);
|
||||
const [data, data2] = json ? JSON.parse(json) : [];
|
||||
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
|
||||
|
||||
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
|
||||
release.title = data?.name || videoData.playerOptions.sceneInfos.sceneTitle;
|
||||
|
||||
release.title = data.name;
|
||||
release.description = data.description;
|
||||
|
||||
// date in data object is not the release date of the scene, but the date the entry was added
|
||||
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
|
||||
const dateString = $('.updatedDate').first().text().trim();
|
||||
const dateMatch = dateString.match(/\d{2,4}-\d{2}-\d{2,4}/)?.[0];
|
||||
release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
|
||||
|
||||
release.director = data.director?.[0].name || data2?.director?.[0].name;
|
||||
release.actors = (data.actor || data2.actor).map(actor => actor.name);
|
||||
const hasTrans = (data.actor || data2.actor).some(actor => actor.gender === 'shemale');
|
||||
if (dateMatch) release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
|
||||
else if (data?.dateCreated) release.date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate();
|
||||
else release.date = videoData.playerOptions.sceneInfos.sceneReleaseDate;
|
||||
|
||||
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
|
||||
if (stars) release.rating = { stars };
|
||||
if (data) {
|
||||
release.description = data.description;
|
||||
release.director = data.director?.[0].name || data2?.director?.[0].name;
|
||||
|
||||
release.duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
|
||||
const actors = data?.actor || data2?.actor || [];
|
||||
const hasTrans = actors.some(actor => actor.gender === 'shemale');
|
||||
release.actors = actors.map(actor => actor.name);
|
||||
|
||||
const rawTags = data.keywords?.split(', ');
|
||||
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
|
||||
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
|
||||
if (stars) release.rating = { stars };
|
||||
|
||||
release.duration = moment.duration(data.duration.slice(2)).asSeconds();
|
||||
|
||||
const rawTags = data.keywords?.split(', ');
|
||||
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
|
||||
}
|
||||
|
||||
release.poster = videoData.picPreview;
|
||||
release.photos = await getPhotos($('.picturesItem a').attr('href'), site);
|
||||
|
||||
const photoLink = $('.picturesItem a').attr('href');
|
||||
if (photoLink) release.photos = await getPhotos(photoLink, site);
|
||||
|
||||
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
|
||||
release.trailer = [
|
||||
@@ -297,6 +300,11 @@ async function fetchApiCredentials(referer) {
|
||||
const body = res.body.toString();
|
||||
|
||||
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
|
||||
|
||||
if (!apiLine) {
|
||||
throw new Error(`Can not use Gamma API for ${referer}`);
|
||||
}
|
||||
|
||||
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
|
||||
const apiData = JSON.parse(apiSerial);
|
||||
|
||||
@@ -321,7 +329,7 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`,
|
||||
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug}`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
@@ -331,7 +339,11 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
return scrapeApiReleases(res.body.results[0].hits, site);
|
||||
if (res.statuscode === 200 && res.body.results?.[0]?.hits) {
|
||||
return scrapeApiReleases(res.body.results[0].hits, site);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function fetchApiUpcoming(site) {
|
||||
@@ -339,14 +351,14 @@ async function fetchApiUpcoming(site) {
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/en/videos/AllCategories/0/${page}`;
|
||||
const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`;
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeAll(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`);
|
||||
const res = await bhttp.get(`${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`);
|
||||
|
||||
return scrapeAll(res.body.toString(), site);
|
||||
}
|
||||
@@ -362,7 +374,7 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
|
||||
requests: [
|
||||
{
|
||||
indexName: 'all_scenes',
|
||||
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
|
||||
params: `query=&filters=sitename:${siteSlug}&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
|
||||
},
|
||||
],
|
||||
}, {
|
||||
@@ -408,7 +420,7 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl) {
|
||||
|
||||
async function fetchApiProfile(actorName, siteSlug) {
|
||||
const actorSlug = encodeURI(actorName);
|
||||
const referer = `https://www.${siteSlug}.com/en/search?query=${actorSlug}&tab=actors`;
|
||||
const referer = `https://www.${siteSlug}.com/en/search`;
|
||||
|
||||
const { apiUrl } = await fetchApiCredentials(referer);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user