DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Using URL or title slug for Whale Member entry IDs for backwards compatability.

This commit is contained in:
DebaucheryLibrarian 2024-06-05 03:05:30 +02:00
parent 129b03eb5a
commit 644226531e
2 changed files with 45 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/scrapers/whalemember.js
View File

@ -3,17 +3,22 @@
const unprint = require('unprint'); const unprint = require('unprint');
const { stripQuery } = require('../utils/url'); const { stripQuery } = require('../utils/url');
const slugify = require('../utils/slugify');
function scrapeLatest(scenes, channel) { function scrapeLatest(scenes, channel) {
return scenes.map(({ query, element }) => { return scenes.map(({ query, _element }) => {
const release = {}; const release = {};
release.url = query.url('[href*="/video"]'); release.url = query.url('[href*="/video"]');
release.entryId = unprint.query.attribute(element, null, 'data-vid'); // release.entryId = unprint.query.attribute(element, null, 'data-vid'); // does not match old videos
release.title = query.content('.video-thumbnail-footer a[href*="/video"]'); release.title = query.content('.video-thumbnail-footer a[href*="/video"]');
release.date = query.date('.actor-list + span', 'MM/DD/YYYY'); release.date = query.date('.actor-list + span', 'MM/DD/YYYY');
release.entryId = release.url
? new URL(release.url).pathname.split('/').at(-1)
: slugify(release.title);
release.actors = query.all('.actor-list a').map((actorEl) => ({ release.actors = query.all('.actor-list a').map((actorEl) => ({
name: unprint.query.content(actorEl), name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null, { origin: channel.url }), url: unprint.query.url(actorEl, null, { origin: channel.url }),
@ -39,10 +44,11 @@ function scrapeLatest(scenes, channel) {
}); });
} }
function scrapeScene({ query }, channel) { function scrapeScene({ query }, { url, entity }) {
const release = {}; const release = {};
release.entryId = query.attribute('div[data-id]', 'data-id'); // release.entryId = query.attribute('div[data-id]', 'data-id');
release.entryId = new URL(url).pathname.split('/').at(-1);
release.title = query.content('.scene-info h1'); release.title = query.content('.scene-info h1');
release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span'); release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span');
@ -51,7 +57,7 @@ function scrapeScene({ query }, channel) {
release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({ release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({
name: unprint.query.content(actorEl), name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null, { origin: channel.url }), url: unprint.query.url(actorEl, null, { origin: entity.url }),
})); }));
release.poster = query.poster('#player-wrapper video'); release.poster = query.poster('#player-wrapper video');

34
src/tools/whalemember-fix.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
// const config = require('config');
const initKnex = require('knex');
// const args = require('yargs').argv;
async function init() {
const knex = initKnex({
client: 'pg',
connection: {
host: '135.125.235.53',
user: 'traxxx',
password: 'YGDdBeXZXE25gKuzh5g7u4RV61G00XP6',
database: 'traxxx',
},
asyncStackTraces: true,
});
const results = await knex('releases')
.select('releases.*')
.leftJoin('entities', 'entities.id', 'releases.entity_id')
.leftJoin('entities as networks', 'networks.id', 'entities.parent_id')
.whereIn('networks.slug', ['whalemember', 'pornpros']);
console.log(results.length);
await knex('releases')
.whereIn('id', results.map((result) => result.id))
.update('entry_id', knex.raw('split_part(releases.url, \'/\', -1)'));
knex.destroy();
}
init();