Added Freaky Fembots to Team Skeet.
This commit is contained in:
DebaucheryLibrarian
100
src/tools/manticore-actors.js
Normal file
100
src/tools/manticore-actors.js
Normal file
@@ -0,0 +1,100 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
const args = require('yargs').argv;
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const searchApi = new manticore.SearchApi(mantiClient);
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
const update = args.update;
|
||||
|
||||
async function fetchActors() {
|
||||
// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
|
||||
const actors = await knex.raw(`
|
||||
SELECT actors.*, date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
|
||||
FROM actors
|
||||
GROUP BY actors.id;
|
||||
`);
|
||||
|
||||
return actors.rows;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
if (update) {
|
||||
await utilsApi.sql('drop table actors');
|
||||
await utilsApi.sql(`create table actors(
|
||||
id int,
|
||||
name text,
|
||||
slug string,
|
||||
gender string,
|
||||
date_of_birth timestamp,
|
||||
country string,
|
||||
has_avatar bool,
|
||||
weight int,
|
||||
height int,
|
||||
cup string,
|
||||
natural_boobs int,
|
||||
penis_length int,
|
||||
penis_girth int
|
||||
)`);
|
||||
|
||||
const actors = await fetchActors();
|
||||
|
||||
const docs = actors.map((actor) => ({
|
||||
insert: {
|
||||
index: 'actors',
|
||||
id: actor.id,
|
||||
doc: {
|
||||
name: actor.name,
|
||||
slug: actor.slug,
|
||||
gender: actor.gender || undefined,
|
||||
date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
|
||||
has_avatar: !!actor.avatar_media_id,
|
||||
country: actor.birth_country_alpha2 || undefined,
|
||||
height: actor.height || undefined,
|
||||
weight: actor.weight || undefined,
|
||||
cup: actor.cup || undefined,
|
||||
natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
|
||||
penis_length: actor.penis_length || undefined,
|
||||
penis_girth: actor.penis_girth || undefined,
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
|
||||
console.log(error);
|
||||
});
|
||||
|
||||
console.log('data', data);
|
||||
knex.destroy();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await searchApi.search({
|
||||
index: 'actors',
|
||||
query: {
|
||||
equals: {
|
||||
has_avatar: 1,
|
||||
},
|
||||
},
|
||||
limit: 3,
|
||||
sort: [{ slug: 'asc' }],
|
||||
});
|
||||
|
||||
console.log(result);
|
||||
console.log(result.hits?.hits);
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
238
src/tools/manticore.js
Normal file
238
src/tools/manticore.js
Normal file
@@ -0,0 +1,238 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
const args = require('yargs').argv;
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const searchApi = new manticore.SearchApi(mantiClient);
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
const update = args.update;
|
||||
|
||||
async function fetchScenes() {
|
||||
const scenes = await knex.raw(`
|
||||
SELECT
|
||||
releases.id AS id,
|
||||
releases.title,
|
||||
releases.created_at,
|
||||
releases.date,
|
||||
releases.entry_id,
|
||||
entities.id as channel_id,
|
||||
entities.slug as channel_slug,
|
||||
entities.name as channel_name,
|
||||
parents.id as network_id,
|
||||
parents.slug as network_slug,
|
||||
parents.name as network_name,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
|
||||
FROM releases
|
||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
GROUP BY releases.id, entities.id, entities.name, entities.slug, entities.alias, parents.id, parents.name, parents.slug, parents.alias;
|
||||
`);
|
||||
|
||||
return scenes.rows;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
if (update) {
|
||||
await utilsApi.sql('drop table scenes');
|
||||
await utilsApi.sql(`create table scenes (
|
||||
id int,
|
||||
title text,
|
||||
entry_id text,
|
||||
channel_id int,
|
||||
channel_name text,
|
||||
channel_slug text,
|
||||
network_id int,
|
||||
network_name text,
|
||||
network_slug text,
|
||||
actor_ids multi,
|
||||
actors text,
|
||||
tag_ids multi,
|
||||
tags text,
|
||||
date timestamp,
|
||||
created_at timestamp,
|
||||
effective_date timestamp
|
||||
)`);
|
||||
|
||||
const scenes = await fetchScenes();
|
||||
|
||||
const docs = scenes.map((scene) => ({
|
||||
insert: {
|
||||
index: 'scenes',
|
||||
id: scene.id,
|
||||
doc: {
|
||||
title: scene.title || undefined,
|
||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
||||
entry_id: scene.entry_id,
|
||||
channel_id: scene.channel_id,
|
||||
channel_slug: scene.channel_slug,
|
||||
channel_name: scene.channel_name,
|
||||
network_id: scene.network_id || undefined,
|
||||
network_slug: scene.network_slug || undefined,
|
||||
network_name: scene.network_name || undefined,
|
||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
||||
tags: scene.tags.map((tag) => tag.f2).join(),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
|
||||
console.log('data', data);
|
||||
}
|
||||
|
||||
/*
|
||||
const searchRequest = new manticore.SearchRequest();
|
||||
|
||||
searchRequest.index = 'scenes';
|
||||
searchRequest.attr_filter = new manticore.InFilter('ratings', [600]);
|
||||
|
||||
const result = await searchApi.search(searchRequest);
|
||||
*/
|
||||
|
||||
const result = await searchApi.search({
|
||||
index: 'scenes',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
/*
|
||||
{
|
||||
match: {
|
||||
'*': 'going-all-out-with-a-gangbang_1080p.mp4',
|
||||
},
|
||||
},
|
||||
*/
|
||||
{
|
||||
range: {
|
||||
effective_date: {
|
||||
lte: Math.round(Date.now() / 1000),
|
||||
},
|
||||
},
|
||||
},
|
||||
/*
|
||||
{
|
||||
equals: {
|
||||
actor_ids: 81930,
|
||||
},
|
||||
},
|
||||
*/
|
||||
],
|
||||
must_not: [
|
||||
{
|
||||
in: {
|
||||
tag_ids: [101, 180, 32],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
limit: 30,
|
||||
sort: [{ effective_date: 'desc' }],
|
||||
/*
|
||||
aggs: {
|
||||
tags: {
|
||||
terms: {
|
||||
field: 'tag_ids',
|
||||
size: 100,
|
||||
},
|
||||
},
|
||||
actors: {
|
||||
terms: {
|
||||
field: 'actor_ids',
|
||||
size: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
*/
|
||||
});
|
||||
|
||||
console.log(result);
|
||||
|
||||
/*
|
||||
if (result.hits) {
|
||||
result.hits.hits.map((hit) => console.log(hit));
|
||||
|
||||
// console.log(result.aggregations.actors);
|
||||
// console.log(result.aggregations.tags);
|
||||
|
||||
const sceneIds = result.hits.hits.map((hit) => hit._id);
|
||||
|
||||
console.time('fetch');
|
||||
|
||||
const [scenes, actors, tags, posters, photos] = await Promise.all([
|
||||
knex('releases').whereIn('id', sceneIds),
|
||||
knex('releases_actors')
|
||||
.whereIn('release_id', sceneIds)
|
||||
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id'),
|
||||
knex('releases_tags')
|
||||
.whereIn('release_id', sceneIds)
|
||||
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id'),
|
||||
knex('releases_posters')
|
||||
.whereIn('release_id', sceneIds)
|
||||
.leftJoin('media', 'media.id', 'releases_posters.media_id'),
|
||||
knex('releases_photos')
|
||||
.whereIn('release_id', sceneIds)
|
||||
.leftJoin('media', 'media.id', 'releases_photos.media_id'),
|
||||
]);
|
||||
|
||||
console.timeEnd('fetch');
|
||||
|
||||
console.time('join');
|
||||
|
||||
const joinedScenes = await knex('releases')
|
||||
.select(knex.raw(`
|
||||
releases.*,
|
||||
COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(json_agg(DISTINCT posters) FILTER (WHERE posters.id IS NOT NULL), '[]') as posters,
|
||||
COALESCE(json_agg(DISTINCT photos) FILTER (WHERE photos.id IS NOT NULL), '[]') as photos
|
||||
`))
|
||||
.whereIn('releases.id', sceneIds)
|
||||
.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
|
||||
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
|
||||
.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
|
||||
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
|
||||
.leftJoin('releases_posters', 'releases_posters.release_id', 'releases.id')
|
||||
.leftJoin('media as posters', 'posters.id', 'releases_posters.media_id')
|
||||
.leftJoin('releases_photos', 'releases_photos.release_id', 'releases.id')
|
||||
.leftJoin('media as photos', 'photos.id', 'releases_photos.media_id')
|
||||
.groupBy('releases.id');
|
||||
|
||||
// console.log(joinedScenes[0]);
|
||||
|
||||
console.timeEnd('join');
|
||||
|
||||
console.log(sceneIds);
|
||||
console.log(scenes);
|
||||
console.log(actors);
|
||||
console.log(tags);
|
||||
console.log(posters);
|
||||
console.log(photos);
|
||||
}
|
||||
*/
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
Reference in New Issue
Block a user