DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updating entity ID for rescraped scenes with network entry IDs enabled.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-06-13 16:49:27 +02:00
parent e9a0700742
commit ab1329dd67
126 changed files with 38 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/scrapers/pervcity.js
View File

@@ -4,6 +4,16 @@ const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
const channelCodes = {
ao: 'analoverdose',
bb: 'bangingbeauties',
cbj: 'chocolatebjs',
oo: 'oraloverdose',
uha: 'upherasshole',
};
const channelRegExp = new RegExp(Object.keys(channelCodes).join('|'), 'i');
function scrapeAll(scenes, entity) {
return scenes.map(({ query }) => {
const release = {};
@@ -42,9 +52,8 @@ function scrapeScene({ query }) {
const trailer = query.q('script')?.textContent.match(/\/trailers\/.+\.mp4/)?.[0];
if (trailer) {
release.trailer = {
src: `https://pervcity.com${trailer}`,
};
release.trailer = `https://pervcity.com${trailer}`;
release.channel = channelCodes[release.trailer.match(channelRegExp)?.[0]];
}
return release;

21
src/store-releases.js
View File

@@ -148,7 +148,8 @@ function attachReleaseIds(releases, storedReleases) {
return null;
}
const id = storedReleaseIdsByEntityIdAndEntryId[release.entity.id]?.[release.entryId];
const id = storedReleaseIdsByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|| storedReleaseIdsByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId];
if (id) {
return {
@@ -193,7 +194,11 @@ async function filterDuplicateReleases(releases) {
const internalUniqueReleases = filterInternalDuplicateReleases(releases);
const duplicateReleaseEntries = await knex('releases')
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map(release => [release.entryId, release.entity.id]));
.whereIn(['entry_id', 'entity_id'], internalUniqueReleases.map(release => [release.entryId, release.entity.id]))
.orWhereIn(['entry_id', 'entity_id'], internalUniqueReleases
// scene IDs shared across network, mark as duplicate so scene can be updated with channel if only available on release day (i.e. Perv City)
.filter(release => release.entity.parent?.parameters?.networkEntryIds)
.map(release => [release.entryId, release.entity.parent.id]));
const duplicateReleasesByEntityIdAndEntryId = duplicateReleaseEntries.reduce((acc, release) => {
if (!acc[release.entity_id]) acc[release.entity_id] = {};
@@ -202,8 +207,11 @@ async function filterDuplicateReleases(releases) {
return acc;
}, {});
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]);
const duplicateReleases = internalUniqueReleases.filter(release => duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
|| duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
const uniqueReleases = internalUniqueReleases.filter(release => !duplicateReleasesByEntityIdAndEntryId[release.entity.id]?.[release.entryId]
&& !duplicateReleasesByEntityIdAndEntryId[release.entity.parent?.id]?.[release.entryId]);
return {
uniqueReleases,
@@ -327,14 +335,15 @@ async function storeScenes(releases) {
UPDATE releases
SET url = COALESCE(new.url, releases.url),
date = COALESCE(new.date, releases.date),
entity_id = COALESCE((new.entity->>'id')::integer, releases.entity_id),
title = COALESCE(new.title, releases.title),
description = COALESCE(new.description, releases.description),
duration = COALESCE(new.duration, releases.duration),
deep = new.url IS NOT NULL,
updated_at = NOW()
FROM json_to_recordset(:scenes)
AS new(id int, url text, date timestamptz, title text, description text, duration integer, deep boolean)
WHERE releases.id = new.id;
AS new(id int, url text, date timestamptz, entity json, title text, description text, duration integer, deep boolean)
WHERE releases.id = new.id
`, {
scenes: JSON.stringify(duplicateReleasesWithId),
});