DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added materialized view for SFW media in hopes of improving media insert performance.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-02-08 04:56:14 +01:00
parent a77ce63548
commit 371e97f695
3 changed files with 43 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
migrations/20260208044729_random_sfw_improvements.js Normal file
View File

@@ -0,0 +1,28 @@
exports.up = async function(knex) {
await knex.schema.createMaterializedView('media_sfw', (view) => {
view.as(knex('media').select('id').where('is_sfw', true));
});
await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT id FROM media_sfw
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
`);
};
exports.down = async function(knex) {
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT id FROM media
WHERE is_sfw = true
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
`);
await knex.schema.dropMaterializedView('media_sfw');
};

2
seeds/04_media.js
View File

@@ -1179,4 +1179,6 @@ exports.seed = (knex) => Promise.resolve()
.whereNotIn('media_id', tagPhotos.map((photo) => photo.id)) .whereNotIn('media_id', tagPhotos.map((photo) => photo.id))
.delete(), .delete(),
]); ]);
await knex.raw('REFRESH MATERIALIZED VIEW media_sfw');
}); });

17
src/scrapers/modelmedia.js
View File

@@ -204,6 +204,13 @@ async function fetchProfileApi(actor, { entity, parameters }) {
return null; return null;
} }
function getBioXPath(field) {
return [
`//span[text()="${field}"]/following-sibling::span`,
`//span[text()="${field}"]/following-sibling::text()`,
];
}
function scrapeProfile({ query }, url) { function scrapeProfile({ query }, url) {
const profile = { url }; const profile = { url };
const avatar = query.img('div[class*="prof-pic"] > img'); const avatar = query.img('div[class*="prof-pic"] > img');
@@ -216,11 +223,13 @@ function scrapeProfile({ query }, url) {
} }
profile.description = query.content('h2') || null; profile.description = query.content('h2') || null;
profile.height = query.number('//span[text()="Height"]/following-sibling::node()[self::span or self::text()]', { match: /(\d+) cm/, matchIndex: 1 });
profile.weight = query.number('//span[text()="Weight"]/following-sibling::node()[self::span or self::text()]', { match: /(\d+) kg/, matchIndex: 1 });
profile.measurements = query.content('//span[text()="Measurements"]/following-sibling::node()[self::span or self::text()]'); // ::node()[self::span or self::text()] not supported by unprint/JSDOM
profile.birthPlace = query.content('//span[text()="Birth Place"]/following-sibling::node()[self::span or self::text()]'); profile.height = query.number(getBioXPath('Height'), { match: /(\d+) cm/, matchIndex: 1 }) || null;
profile.weight = query.number(getBioXPath('Weight'), { match: /(\d+) kg/, matchIndex: 1 }) || null;
profile.measurements = query.content(getBioXPath('Measurements')) || null;
profile.birthPlace = query.content(getBioXPath('Birth Place')) || null;
profile.banner = query.img('div[class*="banner"] > img'); profile.banner = query.img('div[class*="banner"] > img');
profile.photos = query.imgs('#MusModelSwiper img'); profile.photos = query.imgs('#MusModelSwiper img');