Added transfer tool, WIP. Added Savage Gangbang to Kink.
This commit is contained in:
DebaucheryLibrarian
@@ -29,7 +29,9 @@ function scrapeAll(scenes, entity) {
|
||||
|
||||
release.poster = query.img('.adimage');
|
||||
release.photos = query.imgs('.rollover .roll-image', { attribute: 'data-imagesrc' }).map((photo) => [
|
||||
photo.replace('410/', '830/'),
|
||||
photo
|
||||
.replace('410/', '830/')
|
||||
.replace('_thumb', '_full'),
|
||||
photo,
|
||||
]);
|
||||
|
||||
@@ -57,12 +59,12 @@ function scrapeScene({ query }, url, entity) {
|
||||
|
||||
release.director = query.content('.director-name');
|
||||
|
||||
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file');
|
||||
release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', { attribute: 'data-image-file' });
|
||||
release.poster = query.poster();
|
||||
|
||||
release.tags = query.contents('.tag-list a[href*="/tag"]').map((tag) => tag.replace(/,\s*/, ''));
|
||||
|
||||
const trailer = query.attribute('.player span[data-type="trailer-src"]', 'data-url');
|
||||
const trailer = query.attribute('.player span[data-type="trailer-src"]', { attribute: 'data-url' });
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = [
|
||||
|
||||
424
src/tools/transfer.js
Normal file
424
src/tools/transfer.js
Normal file
@@ -0,0 +1,424 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const fs = require('fs').promises;
|
||||
const moment = require('moment');
|
||||
const Promise = require('bluebird');
|
||||
const { nanoid } = require('nanoid/non-secure');
|
||||
const AWS = require('aws-sdk');
|
||||
|
||||
const { graphql } = require('../web/graphql');
|
||||
const knex = require('../knex');
|
||||
const args = require('../argv');
|
||||
|
||||
const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com');
|
||||
|
||||
const s3 = new AWS.S3({
|
||||
// region: 'eu-central-1',
|
||||
endpoint,
|
||||
credentials: {
|
||||
accessKeyId: config.s3.accessKey,
|
||||
secretAccessKey: config.s3.secretKey,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(Object.keys(s3));
|
||||
|
||||
// NOT TRANSFERRED, unutilized on old server: production location, availabile qualities, actor alias for, actor entry id, chapter posters, chapter photos
|
||||
|
||||
const releaseFields = `
|
||||
entryId
|
||||
shootId
|
||||
title
|
||||
url
|
||||
date
|
||||
datePrecision
|
||||
productionDate
|
||||
description
|
||||
duration
|
||||
entity {
|
||||
slug
|
||||
type
|
||||
}
|
||||
studio {
|
||||
slug
|
||||
}
|
||||
actors: releasesActors {
|
||||
actor {
|
||||
name
|
||||
slug
|
||||
entryId
|
||||
entity {
|
||||
slug
|
||||
type
|
||||
}
|
||||
}
|
||||
}
|
||||
directors: releasesDirectors {
|
||||
director {
|
||||
slug
|
||||
entryId
|
||||
entity {
|
||||
slug
|
||||
type
|
||||
}
|
||||
}
|
||||
}
|
||||
tags: releasesTags {
|
||||
tag {
|
||||
slug
|
||||
}
|
||||
}
|
||||
chapters(orderBy: TIME_ASC) {
|
||||
index
|
||||
time
|
||||
duration
|
||||
title
|
||||
description
|
||||
tags: chaptersTags {
|
||||
tag {
|
||||
slug
|
||||
}
|
||||
}
|
||||
}
|
||||
poster: releasesPoster {
|
||||
media {
|
||||
hash
|
||||
path
|
||||
thumbnail
|
||||
lazy
|
||||
s3: isS3
|
||||
mime
|
||||
index
|
||||
width
|
||||
height
|
||||
size
|
||||
source
|
||||
sourcePage
|
||||
}
|
||||
}
|
||||
photos: releasesPhotos {
|
||||
media {
|
||||
hash
|
||||
path
|
||||
thumbnail
|
||||
lazy
|
||||
s3: isS3
|
||||
mime
|
||||
index
|
||||
width
|
||||
height
|
||||
size
|
||||
source
|
||||
sourcePage
|
||||
}
|
||||
}
|
||||
covers: releasesCovers {
|
||||
media {
|
||||
hash
|
||||
path
|
||||
thumbnail
|
||||
lazy
|
||||
s3: isS3
|
||||
mime
|
||||
index
|
||||
width
|
||||
height
|
||||
size
|
||||
source
|
||||
sourcePage
|
||||
}
|
||||
}
|
||||
trailer: releasesTrailer {
|
||||
media {
|
||||
hash
|
||||
path
|
||||
thumbnail
|
||||
lazy
|
||||
s3: isS3
|
||||
mime
|
||||
index
|
||||
width
|
||||
height
|
||||
size
|
||||
source
|
||||
sourcePage
|
||||
}
|
||||
}
|
||||
teaser: releasesTeaser {
|
||||
media {
|
||||
hash
|
||||
path
|
||||
thumbnail
|
||||
lazy
|
||||
s3: isS3
|
||||
mime
|
||||
index
|
||||
width
|
||||
height
|
||||
size
|
||||
source
|
||||
sourcePage
|
||||
}
|
||||
}
|
||||
createdAt
|
||||
`;
|
||||
|
||||
async function save() {
|
||||
const limit = args.limit || 1000;
|
||||
const offset = args.offset || 0;
|
||||
|
||||
const { releases } = await graphql(`
|
||||
query SearchReleases(
|
||||
$limit: Int = 20
|
||||
$offset: Int = 0
|
||||
) {
|
||||
releases(
|
||||
first: $limit
|
||||
offset: $offset
|
||||
orderBy: DATE_DESC
|
||||
) {
|
||||
${releaseFields}
|
||||
}
|
||||
}
|
||||
`, {
|
||||
limit,
|
||||
offset,
|
||||
});
|
||||
|
||||
const curatedReleases = releases.map((release) => ({
|
||||
...release,
|
||||
actors: release.actors.filter(Boolean).map(({ actor }) => actor),
|
||||
directors: release.directors.filter(Boolean).map(({ director }) => director),
|
||||
studio: release.studio?.slug,
|
||||
tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean),
|
||||
chapters: release.chapters.filter(Boolean).map((chapter) => ({
|
||||
...chapter,
|
||||
tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean),
|
||||
})),
|
||||
poster: release.poster?.media,
|
||||
trailer: release.trailer?.media,
|
||||
teaser: release.teaser?.media,
|
||||
photos: release.photos.filter(Boolean).map(({ media }) => media),
|
||||
covers: release.covers.filter(Boolean).map(({ media }) => media),
|
||||
}));
|
||||
|
||||
const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;
|
||||
const serializedData = JSON.stringify(curatedReleases, null, 4);
|
||||
|
||||
await fs.writeFile(filename, serializedData);
|
||||
|
||||
console.log(`Saved ${releases.length} releases to ${filename}`);
|
||||
|
||||
process.exit();
|
||||
}
|
||||
|
||||
async function addReleaseTags(release, context) {
|
||||
if (release.tags.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await knex('releases_tags').insert(release.tags.map((tag) => ({
|
||||
tag_id: context.tagIdsBySlug[tag],
|
||||
release_id: release.id,
|
||||
original_tag: tag,
|
||||
})));
|
||||
}
|
||||
|
||||
async function addNewActor(actor, entity, context) {
|
||||
const [actorId] = await knex('actors')
|
||||
.insert({
|
||||
name: actor.name,
|
||||
slug: actor.slug,
|
||||
entity_id: entity?.id,
|
||||
batch_id: context.batchId,
|
||||
})
|
||||
.returning('id');
|
||||
|
||||
return actorId;
|
||||
}
|
||||
|
||||
async function addReleaseActors(release, context, target = 'actor') {
|
||||
await release[`${target}s`].reduce(async (chain, actor) => {
|
||||
await chain;
|
||||
|
||||
const entity = actor.entity
|
||||
? await knex('entities').where(actor.entity).first()
|
||||
: null;
|
||||
|
||||
if (actor.entity && !entity) {
|
||||
throw new Error(`Actor ${actor.slug} contains non-existent ${release.entity.type} '${release.entity.slug}'`);
|
||||
}
|
||||
|
||||
const existingActor = await knex('actors')
|
||||
.where('slug', actor.slug)
|
||||
.where((builder) => {
|
||||
if (entity) {
|
||||
builder.where('entity_id', entity.id);
|
||||
return;
|
||||
}
|
||||
|
||||
builder.whereNull('entity_id');
|
||||
})
|
||||
.first();
|
||||
|
||||
const actorId = existingActor?.id
|
||||
|| await addNewActor(actor, entity, context);
|
||||
|
||||
await knex(`releases_${target}s`).insert({
|
||||
release_id: release.id,
|
||||
[`${target}_id`]: actorId,
|
||||
});
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function addReleaseDirectors(release, context) {
|
||||
return addReleaseActors(release, context, 'director');
|
||||
}
|
||||
|
||||
async function addReleaseChapters(release, context) {
|
||||
await release.chapters.reduce(async (chain, chapter) => {
|
||||
await chain;
|
||||
|
||||
const [chapterId] = await knex('chapters')
|
||||
.insert({
|
||||
release_id: release.id,
|
||||
index: chapter.index,
|
||||
time: chapter.time,
|
||||
duration: chapter.duration,
|
||||
description: chapter.description,
|
||||
})
|
||||
.returning('id');
|
||||
|
||||
if (chapter.tags.length > 0) {
|
||||
await knex('chapters_tags').insert(chapter.tags.map((tag) => ({
|
||||
tag_id: context.tagIdsBySlug[tag],
|
||||
chapter_id: chapterId,
|
||||
original_tag: tag,
|
||||
})));
|
||||
}
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function addReleaseMedia(medias, release, target) {
|
||||
return Promise.all(medias.filter(Boolean).map(async (media) => {
|
||||
try {
|
||||
const id = nanoid();
|
||||
|
||||
await knex('media').insert({
|
||||
id,
|
||||
hash: media.hash,
|
||||
path: media.path,
|
||||
thumbnail: media.thumbnail,
|
||||
lazy: media.lazy,
|
||||
is_s3: media.s3,
|
||||
index: media.index,
|
||||
mime: media.mime,
|
||||
size: media.size,
|
||||
width: media.width,
|
||||
height: media.height,
|
||||
source: media.source,
|
||||
source_page: media.sourcePage,
|
||||
});
|
||||
|
||||
await knex(`releases_${target}`).insert({
|
||||
release_id: release.id,
|
||||
media_id: id,
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(`Skipped existing media ${media.hash} from ${media.url}: ${error.message}`);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
async function addRelease(release, context) {
|
||||
const existingRelease = await knex('releases')
|
||||
.leftJoin('entities', 'entities.id', 'releases.entity_id')
|
||||
.where('entry_id', release.entryId)
|
||||
.where('entities.slug', release.entity.slug)
|
||||
.where('entities.type', release.entity.type)
|
||||
.first();
|
||||
|
||||
if (existingRelease) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const [entity] = await Promise.all([
|
||||
knex('entities').select('id').where(release.entity).first(),
|
||||
]);
|
||||
|
||||
if (!entity) {
|
||||
throw new Error(`Release contains non-existent ${release.entity.type} '${release.entity.slug}'`);
|
||||
}
|
||||
|
||||
const [releaseId] = await knex('releases')
|
||||
.insert({
|
||||
entry_id: release.entryId,
|
||||
entity_id: entity.id,
|
||||
studio_id: context.studioIdsBySlug[release.studio],
|
||||
shoot_id: release.shootId,
|
||||
url: release.url,
|
||||
title: release.title,
|
||||
slug: release.slug,
|
||||
date: release.date,
|
||||
date_precision: release.datePrecision,
|
||||
production_date: release.productionDate,
|
||||
description: release.description,
|
||||
duration: release.duration,
|
||||
created_batch_id: context.batchId,
|
||||
updated_batch_id: context.batchId,
|
||||
})
|
||||
.returning('id');
|
||||
|
||||
const releaseWithId = { ...release, id: releaseId };
|
||||
|
||||
await Promise.all([
|
||||
addReleaseTags(releaseWithId, context),
|
||||
addReleaseActors(releaseWithId, context),
|
||||
addReleaseDirectors(releaseWithId, context),
|
||||
addReleaseChapters(releaseWithId, context),
|
||||
addReleaseMedia([releaseWithId.poster], releaseWithId, 'posters', context),
|
||||
addReleaseMedia(releaseWithId.photos, releaseWithId, 'photos', context),
|
||||
// addReleaseMedia(releaseWithId.covers, releaseWithId, 'covers', context),
|
||||
]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function load() {
|
||||
const file = await fs.readFile(args.file, 'utf8');
|
||||
const releases = JSON.parse(file);
|
||||
|
||||
const [batchId] = await knex('batches').insert({ comment: `import ${args.file}` }).returning('id');
|
||||
|
||||
const aggTags = Array.from(new Set(releases.flatMap((release) => [...release.tags, ...release.chapters.flatMap((chapter) => chapter.tags)]).filter(Boolean)));
|
||||
const aggStudios = Array.from(new Set(releases.map((release) => release.studio).filter(Boolean)));
|
||||
|
||||
const tags = await knex('tags')
|
||||
.select('id', 'slug')
|
||||
.whereIn('slug', aggTags);
|
||||
|
||||
const studios = await knex('entities')
|
||||
.select('id', 'slug')
|
||||
.where('type', 'studio')
|
||||
.whereIn('slug', aggStudios);
|
||||
|
||||
const tagIdsBySlug = Object.fromEntries(tags.map((tag) => [tag.slug, tag.id]));
|
||||
const studioIdsBySlug = Object.fromEntries(studios.map((studio) => [studio.slug, studio.id]));
|
||||
|
||||
const added = await releases.reduce(async (chain, release) => {
|
||||
const acc = await chain;
|
||||
const isAdded = await addRelease(release, { batchId, tagIdsBySlug, studioIdsBySlug });
|
||||
|
||||
return acc.concat(isAdded);
|
||||
}, Promise.resolve([]));
|
||||
|
||||
console.log(`Loaded ${added.filter(Boolean).length}/${releases.length} scenes in batch ${batchId}`);
|
||||
|
||||
process.exit();
|
||||
}
|
||||
|
||||
({
|
||||
save,
|
||||
load,
|
||||
})[args._]();
|
||||
@@ -1,5 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
/* eslint-disable arrow-body-style */
|
||||
const config = require('config');
|
||||
|
||||
const { postgraphile } = require('postgraphile');
|
||||
@@ -7,7 +8,6 @@ const { postgraphile } = require('postgraphile');
|
||||
const PgConnectionFilterPlugin = require('postgraphile-plugin-connection-filter');
|
||||
const PgSimplifyInflectorPlugin = require('@graphile-contrib/pg-simplify-inflector');
|
||||
const PgOrderByRelatedPlugin = require('@graphile-contrib/pg-order-by-related');
|
||||
const PgAggregatesPlugin = require('@graphile/pg-aggregates').default;
|
||||
|
||||
const { ActorPlugins, SitePlugins, ReleasePlugins, MediaPlugins } = require('./plugins/plugins');
|
||||
|
||||
@@ -19,34 +19,6 @@ async function pgSettings(req) {
|
||||
};
|
||||
}
|
||||
|
||||
// console.log(PgAggregatesPlugin);
|
||||
|
||||
/*
|
||||
const TagsAggregatePlugin = (builder) => {
|
||||
builder.hook('build', (build) => {
|
||||
const pgAggregateSpecs = [
|
||||
{
|
||||
id: 'tags',
|
||||
humanLabel: 'tags',
|
||||
HumanLabel: 'Tags',
|
||||
sqlAggregateWrap: (sqlFrag) => {
|
||||
// console.log('sql frag', sqlFrag);
|
||||
return build.pgSql.fragment`select tag_id from tags where release_id = ${sqlFrag}`;
|
||||
},
|
||||
isSuitableType: (pgType) => {
|
||||
// console.log('pg type', pgType);
|
||||
return pgType.category === 'N';
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
build.pgAggregateGroupBySpecs = pgAggregateSpecs; // eslint-disable-line no-param-reassign
|
||||
|
||||
return build;
|
||||
});
|
||||
};
|
||||
*/
|
||||
|
||||
module.exports = postgraphile(
|
||||
connectionString,
|
||||
'public',
|
||||
@@ -67,13 +39,11 @@ module.exports = postgraphile(
|
||||
appendPlugins: [
|
||||
PgSimplifyInflectorPlugin,
|
||||
PgConnectionFilterPlugin,
|
||||
PgAggregatesPlugin,
|
||||
PgOrderByRelatedPlugin,
|
||||
...ActorPlugins,
|
||||
...SitePlugins,
|
||||
...ReleasePlugins,
|
||||
...MediaPlugins,
|
||||
// TagsAggregatePlugin,
|
||||
],
|
||||
pgSettings,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user