Improved update runner. Improved HTTP module API, added default user agent. Added PornCZ and Czechav logos.
This commit is contained in:
507
src/releases-legacy.js
Normal file
507
src/releases-legacy.js
Normal file
@@ -0,0 +1,507 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const moment = require('moment');
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const argv = require('./argv');
|
||||
const whereOr = require('./utils/where-or');
|
||||
const { associateTags } = require('./tags');
|
||||
const { associateActors, scrapeBasicActors } = require('./actors');
|
||||
const {
|
||||
pluckItems,
|
||||
storeMedia,
|
||||
associateMedia,
|
||||
} = require('./media');
|
||||
const { fetchSites } = require('./sites');
|
||||
const slugify = require('./utils/slugify');
|
||||
const capitalize = require('./utils/capitalize');
|
||||
|
||||
function commonQuery(queryBuilder, {
|
||||
filter = [],
|
||||
after = new Date(0), // January 1970
|
||||
before = new Date(2 ** 44), // May 2109
|
||||
limit = 100,
|
||||
}) {
|
||||
const finalFilter = [].concat(filter); // ensure filter is array
|
||||
|
||||
queryBuilder
|
||||
.leftJoin('sites', 'releases.site_id', 'sites.id')
|
||||
.leftJoin('studios', 'releases.studio_id', 'studios.id')
|
||||
.leftJoin('networks', 'sites.network_id', 'networks.id')
|
||||
.select(
|
||||
'releases.*',
|
||||
'sites.name as site_name', 'sites.slug as site_slug', 'sites.url as site_url', 'sites.network_id', 'sites.parameters as site_parameters',
|
||||
'studios.name as studio_name', 'sites.slug as site_slug', 'studios.url as studio_url',
|
||||
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description',
|
||||
)
|
||||
.whereNotExists((builder) => {
|
||||
// apply tag filters
|
||||
builder
|
||||
.select('*')
|
||||
.from('tags_associated')
|
||||
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
|
||||
.whereIn('tags.slug', finalFilter)
|
||||
.where('tags_associated.domain', 'releases')
|
||||
.whereRaw('tags_associated.target_id = releases.id');
|
||||
})
|
||||
.andWhere('releases.date', '>', after)
|
||||
.andWhere('releases.date', '<=', before)
|
||||
.orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }])
|
||||
.limit(limit);
|
||||
}
|
||||
|
||||
async function curateRelease(release) {
|
||||
const [actors, tags, media] = await Promise.all([
|
||||
knex('actors_associated')
|
||||
.select(
|
||||
'actors.id', 'actors.name', 'actors.gender', 'actors.slug', 'actors.birthdate',
|
||||
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
|
||||
'media.thumbnail as avatar',
|
||||
)
|
||||
.where({ release_id: release.id })
|
||||
.leftJoin('actors', 'actors.id', 'actors_associated.actor_id')
|
||||
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
|
||||
.leftJoin('media', (builder) => {
|
||||
builder
|
||||
.on('media.target_id', 'actors.id')
|
||||
.andOnVal('media.domain', 'actors')
|
||||
.andOnVal('media.index', '0');
|
||||
})
|
||||
.orderBy('actors.gender'),
|
||||
knex('tags_associated')
|
||||
.select('tags.name', 'tags.slug')
|
||||
.where({
|
||||
domain: 'releases',
|
||||
target_id: release.id,
|
||||
})
|
||||
.leftJoin('tags', 'tags.id', 'tags_associated.tag_id')
|
||||
.orderBy('tags.priority', 'desc'),
|
||||
knex('media')
|
||||
.where({
|
||||
target_id: release.id,
|
||||
domain: 'releases',
|
||||
})
|
||||
.orderBy(['role', 'index']),
|
||||
]);
|
||||
|
||||
const curatedRelease = {
|
||||
id: release.id,
|
||||
type: release.type,
|
||||
title: release.title,
|
||||
date: release.date,
|
||||
dateAdded: release.created_at,
|
||||
description: release.description,
|
||||
url: release.url,
|
||||
shootId: release.shoot_id,
|
||||
entryId: release.entry_id,
|
||||
actors: actors.map(actor => ({
|
||||
id: actor.id,
|
||||
slug: actor.slug,
|
||||
name: actor.name,
|
||||
gender: actor.gender,
|
||||
birthdate: actor.birthdate,
|
||||
age: moment().diff(actor.birthdate, 'years'),
|
||||
ageThen: moment(release.date).diff(actor.birthdate, 'years'),
|
||||
avatar: actor.avatar,
|
||||
origin: actor.birth_country_alpha2
|
||||
? {
|
||||
country: {
|
||||
name: actor.birth_country_alias,
|
||||
alpha2: actor.birth_country_alpha2,
|
||||
},
|
||||
}
|
||||
: null,
|
||||
})),
|
||||
director: release.director,
|
||||
tags,
|
||||
duration: release.duration,
|
||||
photos: media.filter(item => item.role === 'photo'),
|
||||
poster: media.filter(item => item.role === 'poster')[0],
|
||||
covers: media.filter(item => item.role === 'cover'),
|
||||
trailer: media.filter(item => item.role === 'trailer')[0],
|
||||
site: {
|
||||
id: release.site_id,
|
||||
name: release.site_name,
|
||||
independent: !!release.site_parameters?.independent,
|
||||
slug: release.site_slug,
|
||||
url: release.site_url,
|
||||
},
|
||||
studio: release.studio_id
|
||||
? {
|
||||
id: release.studio_id,
|
||||
name: release.studio_name,
|
||||
slug: release.studio_slug,
|
||||
url: release.studio_url,
|
||||
}
|
||||
: null,
|
||||
network: {
|
||||
id: release.network_id,
|
||||
name: release.network_name,
|
||||
description: release.network_description,
|
||||
slug: release.network_slug,
|
||||
url: release.network_url,
|
||||
},
|
||||
};
|
||||
|
||||
return curatedRelease;
|
||||
}
|
||||
|
||||
function curateReleases(releases) {
|
||||
return Promise.all(releases.map(async release => curateRelease(release)));
|
||||
}
|
||||
|
||||
async function attachChannelSite(release) {
|
||||
if (!release.site?.isFallback && !release.channel?.force) {
|
||||
return release;
|
||||
}
|
||||
|
||||
if (!release.channel) {
|
||||
throw new Error(`Unable to derive channel site from generic URL: ${release.url}`);
|
||||
}
|
||||
|
||||
const [site] = await fetchSites({
|
||||
name: release.channel.name || release.channel,
|
||||
slug: release.channel.slug || release.channel,
|
||||
});
|
||||
|
||||
if (site) {
|
||||
return {
|
||||
...release,
|
||||
site,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unable to match channel '${release.channel.slug || release.channel}' from generic URL: ${release.url}`);
|
||||
}
|
||||
|
||||
async function attachStudio(release) {
|
||||
if (!release.studio) {
|
||||
return release;
|
||||
}
|
||||
|
||||
const studio = await knex('studios')
|
||||
.where('name', release.studio)
|
||||
.orWhere('slug', release.studio)
|
||||
.orWhere('url', release.studio)
|
||||
.first();
|
||||
|
||||
return {
|
||||
...release,
|
||||
studio,
|
||||
};
|
||||
}
|
||||
|
||||
async function curateReleaseEntry(release, batchId, existingRelease) {
|
||||
const slug = slugify(release.title, {
|
||||
encode: true,
|
||||
limit: config.titleSlugLength,
|
||||
});
|
||||
|
||||
const curatedRelease = {
|
||||
site_id: release.site.id,
|
||||
studio_id: release.studio ? release.studio.id : null,
|
||||
shoot_id: release.shootId || null,
|
||||
entry_id: release.entryId || null,
|
||||
type: release.type,
|
||||
url: release.url,
|
||||
title: release.title,
|
||||
slug,
|
||||
date: release.date,
|
||||
description: release.description,
|
||||
// director: release.director,
|
||||
duration: release.duration,
|
||||
// likes: release.rating && release.rating.likes,
|
||||
// dislikes: release.rating && release.rating.dislikes,
|
||||
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
|
||||
deep: typeof release.deep === 'boolean' ? release.deep : false,
|
||||
deep_url: release.deepUrl,
|
||||
updated_batch_id: batchId,
|
||||
...(!existingRelease && { created_batch_id: batchId }),
|
||||
};
|
||||
|
||||
return curatedRelease;
|
||||
}
|
||||
|
||||
async function fetchReleases(queryObject = {}, options = {}) {
|
||||
const releases = await knex('releases')
|
||||
.modify(commonQuery, options)
|
||||
.andWhere(builder => whereOr(queryObject, 'releases', builder));
|
||||
|
||||
return curateReleases(releases);
|
||||
}
|
||||
|
||||
async function fetchSiteReleases(queryObject, options = {}) {
|
||||
const releases = await knex('releases')
|
||||
.modify(commonQuery, options)
|
||||
.where(builder => whereOr(queryObject, 'sites', builder));
|
||||
|
||||
return curateReleases(releases);
|
||||
}
|
||||
|
||||
async function fetchNetworkReleases(queryObject, options = {}) {
|
||||
const releases = await knex('releases')
|
||||
.modify(commonQuery, options)
|
||||
.where(builder => whereOr(queryObject, 'networks', builder));
|
||||
|
||||
return curateReleases(releases);
|
||||
}
|
||||
|
||||
async function fetchActorReleases(queryObject, options = {}) {
|
||||
const releases = await knex('actors_associated')
|
||||
.leftJoin('releases', 'actors_associated.release_id', 'releases.id')
|
||||
.leftJoin('actors', 'actors_associated.actor_id', 'actors.id')
|
||||
.select(
|
||||
'actors.name as actor_name',
|
||||
)
|
||||
.modify(commonQuery, options)
|
||||
.where(builder => whereOr(queryObject, 'actors', builder));
|
||||
|
||||
return curateReleases(releases);
|
||||
}
|
||||
|
||||
async function fetchTagReleases(queryObject, options = {}) {
|
||||
const releases = await knex('tags_associated')
|
||||
.leftJoin('releases', 'tags_associated.target_id', 'releases.id')
|
||||
.leftJoin('tags', 'tags_associated.tag_id', 'tags.id')
|
||||
.select(
|
||||
'tags.name as tag_name',
|
||||
)
|
||||
.modify(commonQuery, options)
|
||||
.where('tags_associated.domain', 'releases')
|
||||
.where(builder => whereOr(queryObject, 'tags', builder));
|
||||
|
||||
return curateReleases(releases);
|
||||
}
|
||||
|
||||
function accumulateActors(releases) {
|
||||
return releases.reduce((acc, release) => {
|
||||
if (!Array.isArray(release.actors)) return acc;
|
||||
|
||||
release.actors.forEach((actor) => {
|
||||
const actorName = actor.name ? actor.name.trim() : actor.trim();
|
||||
const actorSlug = slugify(actorName);
|
||||
|
||||
if (!actorSlug) return;
|
||||
|
||||
if (!acc[actorSlug]) {
|
||||
acc[actorSlug] = {
|
||||
name: actorName,
|
||||
slug: actorSlug,
|
||||
releaseIds: new Set(),
|
||||
avatars: [],
|
||||
};
|
||||
}
|
||||
|
||||
acc[actorSlug].releaseIds.add(release.id);
|
||||
|
||||
if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info
|
||||
if (actor.avatar) {
|
||||
const avatar = Array.isArray(actor.avatar)
|
||||
? actor.avatar.map(avatarX => ({
|
||||
src: avatarX.src || avatarX,
|
||||
copyright: avatarX.copyright === undefined ? capitalize(release.site?.network?.name) : avatarX.copyright,
|
||||
}))
|
||||
: {
|
||||
src: actor.avatar.src || actor.avatar,
|
||||
copyright: actor.avatar.copyright === undefined ? capitalize(release.site?.network?.name) : actor.avatar.copyright,
|
||||
};
|
||||
|
||||
acc[actorSlug].avatars = acc[actorSlug].avatars.concat([avatar]); // don't flatten fallbacks
|
||||
}
|
||||
});
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
async function storeReleaseAssets(releases) {
|
||||
if (!argv.media) {
|
||||
return;
|
||||
}
|
||||
|
||||
const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
|
||||
const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
|
||||
const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
|
||||
const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
|
||||
const releasePhotosById = releases.reduce((acc, release) => ({
|
||||
...acc,
|
||||
[release.id]: pluckItems(release.photos),
|
||||
}), {});
|
||||
|
||||
if (argv.images && argv.posters) {
|
||||
const posters = await storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster');
|
||||
if (posters) await associateMedia(releasePostersById, posters, 'release', 'poster');
|
||||
}
|
||||
|
||||
if (argv.images && argv.covers) {
|
||||
const covers = await storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover');
|
||||
if (covers) await associateMedia(releaseCoversById, covers, 'release', 'cover');
|
||||
}
|
||||
|
||||
if (argv.images && argv.photos) {
|
||||
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
|
||||
if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo');
|
||||
}
|
||||
|
||||
if (argv.videos && argv.trailers) {
|
||||
const trailers = await storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer');
|
||||
if (trailers) await associateMedia(releaseTrailersById, trailers, 'release', 'trailer');
|
||||
}
|
||||
|
||||
if (argv.videos && argv.teasers) {
|
||||
const teasers = await storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser');
|
||||
if (teasers) await associateMedia(releaseTeasersById, teasers, 'release', 'teaser');
|
||||
}
|
||||
}
|
||||
|
||||
async function updateReleasesSearch(releaseIds) {
|
||||
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
|
||||
|
||||
const documents = await knex.raw(`
|
||||
SELECT
|
||||
releases.id AS release_id,
|
||||
TO_TSVECTOR(
|
||||
'traxxx',
|
||||
releases.title || ' ' ||
|
||||
networks.name || ' ' ||
|
||||
networks.slug || ' ' ||
|
||||
networks.url || ' ' ||
|
||||
sites.name || ' ' ||
|
||||
sites.slug || ' ' ||
|
||||
COALESCE(sites.url, '') || ' ' ||
|
||||
COALESCE(sites.alias, '') || ' ' ||
|
||||
COALESCE(releases.shoot_id, '') || ' ' ||
|
||||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
|
||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
|
||||
) as document
|
||||
FROM releases
|
||||
LEFT JOIN sites ON releases.site_id = sites.id
|
||||
LEFT JOIN networks ON sites.network_id = networks.id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for
|
||||
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
||||
GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url;
|
||||
`, releaseIds && [releaseIds]);
|
||||
|
||||
if (documents.rows?.length > 0) {
|
||||
const query = knex('releases_search').insert(documents.rows).toString();
|
||||
await knex.raw(`${query} ON CONFLICT (release_id) DO UPDATE SET document = EXCLUDED.document`);
|
||||
}
|
||||
}
|
||||
|
||||
async function storeRelease(release, batchId) {
|
||||
if (!release.site) {
|
||||
throw new Error(`Missing site, unable to store "${release.title}" (${release.url})`);
|
||||
}
|
||||
|
||||
if (!release.entryId) {
|
||||
logger.warn(`Missing entry ID, unable to store "${release.title}" (${release.url})`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const existingRelease = await knex('releases')
|
||||
.where({
|
||||
entry_id: release.entryId,
|
||||
site_id: release.site.id,
|
||||
})
|
||||
.first();
|
||||
|
||||
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
|
||||
|
||||
if (existingRelease && !argv.redownload) {
|
||||
return existingRelease;
|
||||
}
|
||||
|
||||
if (existingRelease && argv.redownload) {
|
||||
const [updatedRelease] = await knex('releases')
|
||||
.where('id', existingRelease.id)
|
||||
.update({
|
||||
...existingRelease,
|
||||
...curatedRelease,
|
||||
})
|
||||
.returning('*');
|
||||
|
||||
if (updatedRelease) {
|
||||
await associateTags(release, updatedRelease.id);
|
||||
logger.info(`Updated release "${release.title}" (${existingRelease.id}, ${release.site.name})`);
|
||||
}
|
||||
|
||||
await associateTags(release, existingRelease.id);
|
||||
|
||||
return existingRelease;
|
||||
}
|
||||
|
||||
const [releaseEntry] = await knex('releases')
|
||||
.insert(curatedRelease)
|
||||
.returning('*');
|
||||
|
||||
await associateTags(release, releaseEntry.id);
|
||||
|
||||
logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`);
|
||||
|
||||
return releaseEntry;
|
||||
}
|
||||
|
||||
async function storeReleases(releases) {
|
||||
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
|
||||
|
||||
const storedReleases = await Promise.map(releases, async (release) => {
|
||||
try {
|
||||
const releaseWithChannelSite = await attachChannelSite(release);
|
||||
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
|
||||
const storedRelease = await storeRelease(releaseWithStudio, batchId);
|
||||
|
||||
return storedRelease && {
|
||||
id: storedRelease.id,
|
||||
slug: storedRelease.slug,
|
||||
...releaseWithChannelSite,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
|
||||
return null;
|
||||
}
|
||||
}, {
|
||||
concurrency: 10,
|
||||
}).filter(Boolean);
|
||||
|
||||
logger.info(`Stored ${storedReleases.length} new releases`);
|
||||
|
||||
const actors = accumulateActors(storedReleases);
|
||||
|
||||
await associateActors(actors, storedReleases);
|
||||
|
||||
await Promise.all([
|
||||
// actors need to be stored before generating search
|
||||
updateReleasesSearch(storedReleases.map(release => release.id)),
|
||||
storeReleaseAssets(storedReleases),
|
||||
]);
|
||||
|
||||
if (argv.withProfiles && Object.keys(actors).length > 0) {
|
||||
await scrapeBasicActors();
|
||||
}
|
||||
|
||||
return {
|
||||
releases: storedReleases,
|
||||
actors,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchReleases,
|
||||
fetchActorReleases,
|
||||
fetchSiteReleases,
|
||||
fetchNetworkReleases,
|
||||
fetchTagReleases,
|
||||
storeRelease,
|
||||
storeReleases,
|
||||
updateReleasesSearch,
|
||||
};
|
||||
Reference in New Issue
Block a user