DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Fixed Bang scraper.

This commit is contained in:
DebaucheryLibrarian 2021-02-26 02:52:39 +01:00
parent 16a5d92efe
commit 20129eca5d
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/20190325001339_releases.js
View File

@ -1001,7 +1001,8 @@ exports.up = knex => Promise.resolve()
// allow vim fold // allow vim fold
return knex.raw(` return knex.raw(`
/* We need both the release entries and their search ranking, and PostGraphile does not seem to allow virtual foreign keys on function results. /* We need both the release entries and their search ranking, and PostGraphile does not seem to allow virtual foreign keys on function results.
* Using a view as a proxy for the search results allows us to get both a reference to the releases table, and the ranking */ * Using a view as a proxy for the search results allows us to get both a reference to the releases table, and the ranking.
* A composite type does not seem to be compatible with PostGraphile's @sortable. */
CREATE VIEW releases_search_results AS CREATE VIEW releases_search_results AS
SELECT NULL::integer as id, NULL::real as rank; SELECT NULL::integer as id, NULL::real as rank;

16
src/scrapers/bang.js
View File

@ -83,7 +83,7 @@ async function scrapeScene(scene, entity, options) {
release.poster = getScreenUrl(poster, scene); release.poster = getScreenUrl(poster, scene);
release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene)); release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene));
if (options.includePhotos) { if (options?.includePhotos) {
const photos = await fetchPhotos(scene); const photos = await fetchPhotos(scene);
if (photos?.length > 0) { if (photos?.length > 0) {
@ -100,11 +100,11 @@ async function scrapeScene(scene, entity, options) {
return release; return release;
} }
function scrapeAll(scenes) { function scrapeAll(scenes, entity) {
return scenes.map(({ _source: scene }) => scrapeScene(scene)); return scenes.map(({ _source: scene }) => scrapeScene(scene, entity));
} }
async function fetchActorReleases(actor) { async function fetchActorReleases(actor, entity) {
const res = await http.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, { const res = await http.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50, size: 50,
query: { query: {
@ -158,10 +158,10 @@ async function fetchActorReleases(actor) {
}, },
}); });
return scrapeAll(res.body.hits.hits); return scrapeAll(res.body.hits.hits, entity);
} }
async function scrapeProfile(actor, include) { async function scrapeProfile(actor, entity, include) {
const profile = {}; const profile = {};
profile.aliases = actor.aliases; profile.aliases = actor.aliases;
@ -193,7 +193,7 @@ async function scrapeProfile(actor, include) {
if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`; if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`;
if (include.releases) { if (include.releases) {
profile.releases = await fetchActorReleases(actor); profile.releases = await fetchActorReleases(actor, entity);
} }
return profile; return profile;
@ -402,7 +402,7 @@ async function fetchProfile({ name: actorName }, context, include) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase()); const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) { if (actor) {
return scrapeProfile(actor._source, include); return scrapeProfile(actor._source, context.entity, include);
} }
return null; return null;