Using paginated full text search for movies, combined actor search and fetch to allow combining search with filters.

This commit is contained in:
DebaucheryLibrarian 2021-08-22 22:25:20 +02:00
parent e0905ab8fc
commit 6a8c9d89cb
21 changed files with 163 additions and 196 deletions

View File

@ -7,9 +7,48 @@
ref="filters"
class="filters"
>
<SearchBar :placeholder="`Search ${totalCount} actors`" />
<div class="filters-row">
<ul class="genders nolist">
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'all', letter, pageNumber: 1 }, query: $route.query }"
:class="{ selected: gender === 'all' }"
class="gender-link all"
>all</router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'female', letter, pageNumber: 1 }, query: $route.query }"
:class="{ selected: gender === 'female' }"
class="gender-link female"
><Gender gender="female" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'male', letter, pageNumber: 1 }, query: $route.query }"
:class="{ selected: gender === 'male' }"
class="gender-link male"
replace
><Gender gender="male" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'trans', letter, pageNumber: 1 }, query: $route.query }"
:class="{ selected: gender === 'trans' }"
class="gender-link transsexual"
replace
><Gender gender="transsexual" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'other', letter, pageNumber: 1 }, query: $route.query }"
:class="{ selected: gender === 'other' }"
class="gender-link other"
replace
><Icon icon="question5" /></router-link>
</li>
</ul>
<ul class="nolist">
<li>
<Tooltip class="filter boobs">
@ -156,48 +195,9 @@
</Tooltip>
</li>
</ul>
<ul class="genders nolist">
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'all', letter, pageNumber: 1 } }"
:class="{ selected: gender === 'all' }"
class="gender-link all"
>all</router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'female', letter, pageNumber: 1 } }"
:class="{ selected: gender === 'female' }"
class="gender-link female"
><Gender gender="female" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'male', letter, pageNumber: 1 } }"
:class="{ selected: gender === 'male' }"
class="gender-link male"
replace
><Gender gender="male" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'trans', letter, pageNumber: 1 } }"
:class="{ selected: gender === 'trans' }"
class="gender-link transsexual"
replace
><Gender gender="transsexual" /></router-link>
</li>
<li class="gender">
<router-link
:to="{ name: 'actors', params: { gender: 'other', letter, pageNumber: 1 } }"
:class="{ selected: gender === 'other' }"
class="gender-link other"
replace
><Icon icon="question5" /></router-link>
</li>
</ul>
</div>
<SearchBar :placeholder="`Search ${totalCount} actors`" />
</nav>
<div class="tiles">
@ -247,6 +247,7 @@ function updateFilters() {
w: this.weightRequired ? this.weight.join(',') : undefined,
age: this.ageRequired ? this.age.join(',') : undefined,
dob: this.dobRequired ? this.dob : undefined,
query: this.$route.query.query,
},
});
}
@ -260,17 +261,12 @@ function updateValue(prop, value, load = true) {
}
async function fetchActors(scroll) {
if (this.$route.query.query) {
await this.searchActors();
return;
}
const curatedGender = this.gender.replace('trans', 'transsexual');
const { actors, totalCount } = await this.$store.dispatch('fetchActors', {
limit: this.limit,
pageNumber: Number(this.$route.params.pageNumber) || 1,
letter: this.letter.replace('all', ''),
query: this.$route.query.query,
gender: curatedGender === 'other' ? null : curatedGender,
age: this.ageRequired && this.age,
dob: this.dobRequired && this.dob,
@ -289,22 +285,6 @@ async function fetchActors(scroll) {
}
}
async function searchActors(scroll) {
const actors = await this.$store.dispatch('searchActors', {
minLength: 1,
query: this.$route.query.query,
limit: this.limit,
});
this.actors = actors;
this.totalCount = actors.length;
if (scroll) {
this.$refs.content.scrollTop = 0;
// this.$refs.filter?.scrollIntoView();
}
}
function letter() {
return this.$route.params.letter || 'all';
}
@ -366,7 +346,6 @@ export default {
mounted,
methods: {
fetchActors,
searchActors,
updateFilters,
updateValue,
},
@ -419,6 +398,7 @@ export default {
.search {
width: 0;
justify-content: flex-end;
flex-grow: 1;
box-sizing: border-box;
padding: 0 1rem;
@ -435,10 +415,6 @@ export default {
margin: 1rem 0 .5rem 0;
}
.filters-row {
flex-grow: 1;
}
.filters-row,
.filter {
padding: 0 1rem;
@ -447,7 +423,7 @@ export default {
.genders {
display: flex;
flex-shrink: 0;
padding: 0 0 0 .5rem;
padding: 0 .5rem 0 0;
}
.letter,
@ -690,7 +666,7 @@ export default {
@media(max-width: $breakpoint-kilo) {
.filters {
flex-direction: column;
flex-direction: column-reverse;
}
::v-deep(.search) {
@ -702,7 +678,7 @@ export default {
@media(max-width: $breakpoint) {
.filters-row {
flex-direction: column-reverse;
flex-direction: column;
}
.genders {

View File

@ -45,13 +45,14 @@ async function fetchMovies() {
}
async function searchMovies() {
const movies = await this.$store.dispatch('searchMovies', {
const { movies, totalCount } = await this.$store.dispatch('searchMovies', {
query: this.$route.query.query,
limit: this.limit,
pageNumber: this.$route.params.pageNumber,
});
this.movies = movies;
this.totalCount = movies.length;
this.totalCount = totalCount;
}
async function mounted() {

View File

@ -22,6 +22,10 @@ function search() {
this.$router.replace({ query: { query: this.query || undefined } });
}
function resetQuery() {
this.query = this.$route.query.query || null;
}
export default {
props: {
placeholder: {
@ -34,6 +38,9 @@ export default {
query: this.$route.query.query || null,
};
},
watch: {
$route: resetQuery,
},
methods: {
search,
},

View File

@ -289,7 +289,7 @@ function initActorActions(store, router) {
async function fetchActors({ _commit }, {
limit = 10,
pageNumber = 1,
letter,
query,
gender,
age,
dob,
@ -332,24 +332,23 @@ function initActorActions(store, router) {
const { connection: { actors, totalCount } } = await graphql(`
query Actors(
$query: String
$limit: Int,
$offset: Int = 0,
$letter: String! = "",
$naturalBoobs: Boolean,
$userId: Int,
$hasAuth: Boolean!,
) {
connection: actorsConnection(
connection: searchActorsConnection(
query: $query
first: $limit,
offset: $offset
orderBy: NAME_ASC
minLength: 0
filter: {
aliasFor: {
isNull: true
}
name: {
startsWith: $letter
}
${genderFilter}
${ageFilter}
${dobFilter}
@ -410,9 +409,9 @@ function initActorActions(store, router) {
}
}
`, {
query,
offset: Math.max(0, (pageNumber - 1)) * limit,
limit,
letter,
naturalBoobs,
hasAuth: !!store.state.auth.user,
userId: store.state.auth.user?.id,
@ -424,96 +423,6 @@ function initActorActions(store, router) {
};
}
async function searchActors({ _commit }, { query, limit = 20, minLength = 2 }) {
const { actors } = await graphql(`
query SearchActors(
$query: String!
$limit: Int = 20
$minLength: Int = 2
$hasAuth: Boolean!
$userId: Int
) {
actors: searchActors(
search: $query,
minLength: $minLength
first: $limit
orderBy: NAME_ASC
) {
id
name
slug
dateOfBirth
ageFromBirth
ageAtDeath
dateOfBirth
dateOfDeath
gender
aliasFor: actorByAliasFor {
id
name
slug
age
ageAtDeath
dateOfBirth
dateOfDeath
gender
entity {
id
name
slug
}
avatar: avatarMedia {
id
path
thumbnail
lazy
isS3
width
height
comment
credit
}
birthCountry: countryByBirthCountryAlpha2 {
alpha2
name
alias
}
}
entity {
id
name
slug
}
avatar: avatarMedia {
id
path
thumbnail
lazy
isS3
width
height
comment
credit
}
birthCountry: countryByBirthCountryAlpha2 {
alpha2
name
alias
}
${actorStashesFields}
}
}
`, {
query,
limit,
minLength,
hasAuth: !!store.state.auth.user,
userId: store.state.auth.user?.id,
});
return actors.map(actor => curateActor(actor));
}
async function fetchActorReleases({ _commit }, actorId) {
const releases = await get(`/actors/${actorId}/releases`, {
filter: store.state.ui.filter,
@ -528,7 +437,6 @@ function initActorActions(store, router) {
fetchActorById,
fetchActors,
fetchActorReleases,
searchActors,
};
}

View File

@ -100,26 +100,37 @@ function initReleasesActions(store, router) {
};
}
async function searchMovies({ _commit }, { query, limit = 20 }) {
const { movies } = await graphql(`
async function searchMovies({ _commit }, { query, limit = 20, pageNumber = 1 }) {
const { connection: { movies, totalCount } } = await graphql(`
query SearchMovies(
$query: String!
$limit:Int = 20,
$limit:Int = 20
$offset:Int = 0
) {
movies: searchMovies(
connection: searchMoviesConnection(
query: $query
minLength: 1
first: $limit
offset: $offset
orderBy: RANK_DESC
) {
movies: nodes {
movie {
${movieFields}
}
}
totalCount
}
}
`, {
query,
limit,
offset: Math.max(0, (pageNumber - 1)) * limit,
});
return movies.map(movie => curateRelease(movie));
return {
movies: movies.map(({ movie }) => curateRelease(movie)),
totalCount,
};
}
async function fetchMovieById({ _commit }, movieId) {

View File

@ -915,6 +915,12 @@ exports.up = knex => Promise.resolve()
table.unique('movie_id');
}))
.then(() => knex.schema.createTable('movies_search', (table) => {
table.integer('movie_id', 16)
.references('id')
.inTable('movies')
.onDelete('cascade');
}))
.then(() => knex.schema.createTable('chapters', (table) => {
table.increments('id', 16);
@ -1369,6 +1375,7 @@ exports.up = knex => Promise.resolve()
// allow vim fold
return knex.raw(`
ALTER TABLE releases_search ADD COLUMN document tsvector;
ALTER TABLE movies_search ADD COLUMN document tsvector;
/* allow scenes without dates to be mixed inbetween scenes with dates */
ALTER TABLE releases
@ -1389,14 +1396,16 @@ exports.up = knex => Promise.resolve()
CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL;
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
`);
})
// FUNCTIONS
.then(() => { // eslint-disable-line arrow-body-style
// allow vim fold
return knex.raw(`
CREATE EXTENSION pg_trgm;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE FUNCTION current_user_id() RETURNS INTEGER AS $$
/* if the user ID is undefined, the adapter will pass it as a string, which cannot be cast as NULL by ::integer */
@ -1407,6 +1416,7 @@ exports.up = knex => Promise.resolve()
* Using a table as a proxy for the search results allows us to get both a reference to the releases table, and the ranking.
* A composite type does not seem to be compatible with PostGraphile's @sortable, and a view does not allow for many native constraints */
CREATE TABLE releases_search_results (release_id integer, rank real, FOREIGN KEY (release_id) REFERENCES releases (id));
CREATE TABLE movies_search_results (movie_id integer, rank real, FOREIGN KEY (movie_id) REFERENCES movies (id));
CREATE FUNCTION search_releases(query text) RETURNS SETOF releases_search_results AS $$
SELECT releases.id, ranks.rank FROM (
@ -1420,10 +1430,16 @@ exports.up = knex => Promise.resolve()
ORDER BY ranks.rank DESC;
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION search_movies(query text, min_length smallint DEFAULT 2) RETURNS SETOF movies AS $$
SELECT * FROM movies
WHERE length(query) >= min_length
AND title ILIKE ('%' || TRIM(query) || '%')
CREATE FUNCTION search_movies(query text) RETURNS SETOF movies_search_results AS $$
SELECT movies.id, ranks.rank FROM (
SELECT
movies_search.movie_id,
ts_rank(movies_search.document, to_tsquery('english', array_to_string(array(SELECT * FROM regexp_matches(query, '[A-Za-zÀ-ÖØ-öø-ÿ0-9]+', 'g')), '|'))) AS rank
FROM movies_search
) ranks
LEFT JOIN movies ON movies.id = ranks.movie_id
WHERE ranks.rank > 0
ORDER BY ranks.rank DESC;
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$
@ -1436,13 +1452,14 @@ exports.up = knex => Promise.resolve()
url ILIKE ('%' || search || '%')
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION search_actors(search text, min_length smallint DEFAULT 2) RETURNS SETOF actors AS $$
CREATE FUNCTION search_actors(query text, min_length smallint DEFAULT 2) RETURNS SETOF actors AS $$
SELECT * FROM actors
WHERE length(search) >= min_length
WHERE query IS NULL
OR length(query) >= min_length
AND CASE
WHEN length(search) > 1
THEN name ILIKE ('%' || TRIM(search) || '%')
ELSE name ILIKE (TRIM(search) || '%')
WHEN length(query) > 1 THEN name ILIKE ('%' || TRIM(query) || '%')
WHEN length(query) = 1 THEN name ILIKE (TRIM(query) || '%')
ELSE true
END
$$ LANGUAGE SQL STABLE;
@ -1750,6 +1767,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS releases_teasers CASCADE;
DROP TABLE IF EXISTS releases_tags CASCADE;
DROP TABLE IF EXISTS releases_search CASCADE;
DROP TABLE IF EXISTS movies_search CASCADE;
DROP TABLE IF EXISTS movies_scenes CASCADE;
DROP TABLE IF EXISTS movies_covers CASCADE;
@ -1848,5 +1866,6 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP FUNCTION IF EXISTS get_random_campaign;
DROP TABLE IF EXISTS releases_search_results;
DROP TABLE IF EXISTS movies_search_results;
`);
};

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 733 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 686 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 697 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

View File

@ -597,6 +597,7 @@ const tagMedia = [
['airtight', 7, 'Lana Rhoades in "Gangbang Me 3"', 'hardx'],
['airtight', 6, 'Remy Lacroix in "Ass Worship 14"', 'julesjordan'],
['airtight', 'anissa_kate_legalporno', 'Anissa Kate in GP1962', 'legalporno'],
['airtight', 'tory_lane_bigtitsatwork', 'Tory Lane in "I\'m Your Christmas Bonus"', 'bigtitsatwork'],
['airtight', 11, 'Malena Nazionale in "Rocco\'s Perverted Secretaries 2: Italian Edition"', 'roccosiffredi'],
['airtight', 3, 'Anita Bellini in "Triple Dick Gangbang"', 'handsonhardcore'],
['airtight', 'venera_maxima_legalporno', 'Venera Maxima in LegalPorno SZ2645', 'legalporno'],
@ -628,6 +629,8 @@ const tagMedia = [
['anal-fingering', 1, 'Cherry Kiss, Veronica Leal and Sybil in "Capture This"', 'lezcuties'],
['anal-fingering', 2, 'Aidra Fox and Cassidy Klein in "Lesbian Anal Yoga"', 'lesbianx'],
['anal-fisting', 1, 'Jane Wilde fisting Alexis Tae in "Jane Wilde is AGAPE"', 'evilangel'],
['anal-prolapse', 'alysa_isabella_clark_evilangel_1', 'Alysa and Isabella Clark in "Anal Buffet 6"', 'evilangel'],
['anal-prolapse', 'alysa_isabella_clark_evilangel', 'Alysa and Isabella Clark in "Anal Buffet 6"', 'evilangel'],
['asian', 0, 'Vina Sky', 'eroticax', 'eroticax'],
['ass-eating', 2, 'Tina Kay, Lexi Layo and Tiffany Tatum in "Ballerina Beauties"', 'lezcuties'],
['ass-eating', 5, 'Remy LaCroix and Abigail Mac', 'allgirlmassage'],
@ -981,9 +984,10 @@ const tagMedia = [
['titty-fucking', 4, 'Set 5532', 'tugjobs'],
['titty-fucking', 3, 'Anna Bell Peaks in "Ringing Her Bell"', 'milfvr'],
['titty-fucking', 1, 'Chloe Lamour', 'ddfbusty'],
['toy-anal', 3, 'Kelly and Leona in "Sleeping Over"', 'lezcuties'],
['toy-anal', 'kelsi_monroe_roxy_raye_evilangel', 'Kelsi Monroe and Roxy Raye', 'evilangel'],
['toy-anal', 'ember_snow_jane_wilde_lesbianx', 'Ember Snow and Jane Wilde in "Ember\'s Wilde Ride"', 'lesbianx'],
['toy-anal', 1, 'Nina North and Cassidy Klein in "Nina\'s First Lesbian Anal"', 'lesbianx'],
['toy-anal', 3, 'Kelly and Leona in "Sleeping Over"', 'lezcuties'],
['toy-anal', 2, 'Denise, Irina and Laki in "Sexy Slumber"', 'lezcuties'],
['toy-anal', 0, 'Kira Noir in 1225', 'inthecrack'],
['toy-dp', 1, 'Krissy Lynn and London River in "Lesbian DP Workout"', 'lesbianx'],

View File

@ -10,7 +10,7 @@ const http = require('./utils/http');
const knex = require('./knex');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateReleasesSearch, associateMovieScenes } = require('./store-releases');
const { storeScenes, storeMovies, updateSceneSearch, updateMovieSearch, associateMovieScenes } = require('./store-releases');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
@ -35,7 +35,10 @@ async function init() {
}
if (argv.updateSearch) {
await updateReleasesSearch();
await Promise.all([
updateSceneSearch(),
updateMovieSearch(),
]);
}
if (argv.interpolateProfiles) {

View File

@ -220,7 +220,7 @@ async function filterDuplicateReleases(releases) {
};
}
async function updateReleasesSearch(releaseIds) {
async function updateSceneSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
@ -355,7 +355,7 @@ async function storeScenes(releases) {
]);
await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
await updateReleasesSearch(releasesWithId.map(release => release.id));
await updateSceneSearch(releasesWithId.map(release => release.id));
// media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId);
@ -400,6 +400,43 @@ async function associateMovieScenes(movies, movieScenes) {
await bulkInsert('movies_scenes', associations, false);
}
async function updateMovieSearch(movieIds) {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } movies`);
const documents = await knex.raw(`
SELECT
movies.id AS movie_id,
TO_TSVECTOR(
'english',
COALESCE(movies.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(movies.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM movies
LEFT JOIN entities ON movies.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
LEFT JOIN releases ON releases.id = movies_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = movies_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
GROUP BY movies.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert('movies_search', documents.rows, ['movie_id']);
}
}
async function storeMovies(movies) {
if (!movies || movies.length === 0) {
return [];
@ -422,5 +459,6 @@ module.exports = {
associateMovieScenes,
storeScenes,
storeMovies,
updateReleasesSearch,
updateSceneSearch,
updateMovieSearch,
};