diff --git a/migrations/20240104234936_actors_metadata.js b/migrations/20240104234936_actors_metadata.js index 47c99f5b9..87e6a4228 100644 --- a/migrations/20240104234936_actors_metadata.js +++ b/migrations/20240104234936_actors_metadata.js @@ -1,3 +1,5 @@ +const config = require('config'); + exports.up = async function up(knex) { await knex.raw(` CREATE MATERIALIZED VIEW actors_meta AS ( @@ -40,9 +42,15 @@ exports.up = async function up(knex) { LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = movies_scenes.scene_id GROUP BY movies.id - ) + ) AS meta ); - `); + + GRANT ALL ON actors_meta TO :visitor; + GRANT ALL ON scenes_meta TO :visitor; + GRANT ALL ON movies_meta TO :visitor; + `, { + visitor: knex.raw(config.database.query.user), + }); }; exports.down = async function down(knex) { diff --git a/src/scrapers/whalemember.js b/src/scrapers/whalemember.js index ac311f389..af5f0fa2b 100755 --- a/src/scrapers/whalemember.js +++ b/src/scrapers/whalemember.js @@ -1,7 +1,6 @@ 'use strict'; const { JSDOM } = require('jsdom'); -const unprint = require('unprint'); const moment = require('moment'); const http = require('../utils/http'); @@ -51,16 +50,14 @@ function scrapeLatest(html, site) { }; } - console.log(release); - return release; }); } -function scrapeScene({ query }, { url, entity }) { - const release = {}; +function scrapeScene(html, site, url) { + const { document } = new JSDOM(html).window; + const release = { site }; -<<<<<<< Updated upstream const scene = document.querySelector('#t2019-2col'); release.url = url; @@ -81,46 +78,42 @@ function scrapeScene({ query }, { url, entity }) { release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), (el) => ({ src: (/^http/.test(el.src) ? el.src : `https:${el.src}`), referer: site.url, -======= - release.entryId = new URL(url).pathname.match('video/(.*)')?.[1]; - - release.title = query.content('.scene-info .text-2xl'); - release.description = query.content('.scene-info .space-x-4 span'); - - release.actors = query.all('.scene-info .link-list-with-commas a').map((el) => ({ - name: unprint.query.content(el), - url: unprint.query.url(el, null, { origin: entity.url }), - })); - - // unreliable CDN - release.photos = query.imgs('#trailer_player .flex-row img').filter((src) => src?.includes('cdn.')).map((src) => ({ - src, - referer: entity.url, ->>>>>>> Stashed changes attempts: 5, interval: 5000, concurrency: 1, })); - release.poster = { - src: query.img('#no-player-image') || query.poster('#player'), - referer: entity.url, - attempts: 5, - interval: 5000, - concurrency: 1, - }; + const posterEl = scene.querySelector('#no-player-image'); + const videoEl = scene.querySelector('video'); + const trailerEl = scene.querySelector('#t2019-video source'); - if (query.exists('#player source')) { - release.trailer = { - src: query.video('#player source'), - referer: entity.url, + if (posterEl) { + release.poster = { + src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`, + referer: site.url, + attempts: 5, + interval: 5000, + concurrency: 1, + }; + } else if (videoEl) { + release.poster = { + src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`, + referer: site.url, attempts: 5, interval: 5000, concurrency: 1, }; } - console.log(release); + if (trailerEl) { + release.trailer = { + src: trailerEl.src, + referer: site.url, + attempts: 5, + interval: 5000, + concurrency: 1, + }; + } return release; } @@ -136,10 +129,17 @@ async function fetchLatest(site, page = 1) { return []; } +async function fetchScene(url, site) { + const res = await http.get(url); + + if (res.statusCode === 200) { + return scrapeScene(res.body.toString(), site, url); + } + + return null; +} + module.exports = { fetchLatest, - scrapeScene: { - scraper: scrapeScene, - unprint: true, - }, + fetchScene, };