From 4ede0724a7bf341b208c3bca3655636861af59c2 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Mon, 19 Oct 2026 07:35:03 +0200 Subject: [PATCH] Migrated remaining scrapers from qu/http to unprint, likely fixing TeamSkeet and PinkyXXX (verification needed). --- src/deep.js | 31 +- src/media.js | 6 +- src/scrapers/actors.js | 3 - src/scrapers/americanpornstar.js | 28 +- src/scrapers/analvids.js | 174 ------ src/scrapers/analvids_legacy.js | 203 ------- src/scrapers/assylum.js | 57 +- src/scrapers/badoink.js | 66 +-- src/scrapers/czechav.js | 34 +- src/scrapers/elevatedx.js | 101 ++-- src/scrapers/insex.js | 83 +-- src/scrapers/jesseloadsmonsterfacials.js | 47 +- src/scrapers/perfectgonzo.js | 30 +- src/scrapers/pinkyxxx.js | 85 +-- src/scrapers/releases.js | 2 - src/scrapers/rickysroom.js | 36 +- src/scrapers/teamskeet.js | 7 +- src/scrapers/topwebmodels.js | 144 ----- src/store-releases.js | 3 +- src/tools/dogfart.js | 76 --- src/utils/media.js | 110 ---- src/utils/q.js | 5 - src/utils/qu.js | 682 ----------------------- src/utils/timeout.js | 20 - 24 files changed, 301 insertions(+), 1732 deletions(-) delete mode 100644 src/scrapers/analvids.js delete mode 100755 src/scrapers/analvids_legacy.js delete mode 100755 src/scrapers/topwebmodels.js delete mode 100755 src/tools/dogfart.js delete mode 100755 src/utils/media.js delete mode 100755 src/utils/q.js delete mode 100755 src/utils/qu.js delete mode 100755 src/utils/timeout.js diff --git a/src/deep.js b/src/deep.js index 28d163d4..7df131c8 100755 --- a/src/deep.js +++ b/src/deep.js @@ -9,7 +9,6 @@ const argv = require('./argv'); const include = require('./utils/argv-include')(argv); const { fetchReleaseEntities, urlToHostname } = require('./entities'); const logger = require('./logger')(__filename); -const qu = require('./utils/qu'); const getRecursiveParameters = require('./utils/get-recursive-parameters'); const windows = require('./utils/http-windows'); @@ -85,35 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc } if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) { - /* - if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) { - return fetchUnprintScene(scraper, url, entity, baseRelease, options, type); - } - */ - - if (!scraper.deprecated) { - return fetchUnprintScene(scraper, url, entity, baseRelease, options, type); - } - - const session = qu.session(); - - const res = await qu.get(url, null, null, { - session, - rejectUnauthorized: false, - }); - - const cookie = await session._sessionOptions.cookieJar.get(url); - - if (res.ok) { - return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, { - session, - headers: res.headers, - cookieJar: session._sessionOptions.cookieJar, - cookie, - }); - } - - return res.status; + return fetchUnprintScene(scraper, url, entity, baseRelease, options, type); } return null; diff --git a/src/media.js b/src/media.js index a50a4f54..b3e0341a 100755 --- a/src/media.js +++ b/src/media.js @@ -15,6 +15,7 @@ const sharp = require('sharp'); const blake2 = require('blake2'); const { format } = require('date-fns'); const taskQueue = require('promise-task-queue'); +const unprint = require('unprint'); const { Upload } = require('@aws-sdk/lib-storage'); const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); @@ -25,7 +26,6 @@ const knex = require('./knex'); const http = require('./utils/http'); const batchInsert = require('./utils/batch-insert'); const chunk = require('./utils/chunk'); -const { get } = require('./utils/qu'); const { fetchEntityReleaseIds } = require('./entity-releases'); const streamQueue = taskQueue(); @@ -383,10 +383,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) { }; } - const res = await get(baseSource.url); + const res = await unprint.get(baseSource.url); if (res.ok) { - const src = await baseSource.extract(res.item); + const src = await baseSource.extract(res.context); return { ...baseSource, diff --git a/src/scrapers/actors.js b/src/scrapers/actors.js index eba2fa62..bfbf597c 100644 --- a/src/scrapers/actors.js +++ b/src/scrapers/actors.js @@ -29,7 +29,6 @@ const karups = require('./karups'); const kellymadison = require('./kellymadison'); const kink = require('./kink'); const mariskax = require('./mariskax'); -// const analvids = require('./analvids'); const pornbox = require('./pornbox'); const littlecapricedreams = require('./littlecapricedreams'); const loveherfilms = require('./loveherfilms'); @@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld'); const testedefudelidade = require('./testedefudelidade'); const theflourish = require('./theflourish'); const tokyohot = require('./tokyohot'); -// const topwebmodels = require('./topwebmodels'); const virtualtaboo = require('./virtualtaboo'); const vixen = require('./vixen'); const wankzvr = require('./wankzvr'); @@ -229,7 +227,6 @@ module.exports = { boyfun: karups, kellymadison, '8kmembers': kellymadison, - // analvids, analvids: pornbox, littlecapricedreams, mariskax, diff --git a/src/scrapers/americanpornstar.js b/src/scrapers/americanpornstar.js index 124689c7..67bfaea7 100755 --- a/src/scrapers/americanpornstar.js +++ b/src/scrapers/americanpornstar.js @@ -1,45 +1,45 @@ 'use strict'; -const { fetchLatest, fetchProfile } = require('./julesjordan'); +const unprint = require('unprint'); -const qu = require('../utils/q'); +const { fetchLatest, fetchProfile } = require('./julesjordan'); const slugify = require('../utils/slugify'); function scrapeScene({ query }, channel) { const release = {}; - const title = query.cnt('.update_title', false); + const title = query.content('.update_title', { trim: false }); release.title = title.trim(); - release.description = query.cnt('.latest_update_description'); + release.description = query.content('.latest_update_description'); release.date = query.date('.update_date', 'MM/DD/YYYY'); release.actors = query.all('.tour_update_models a').map((actorEl) => ({ - name: query.cnt(actorEl), - url: query.url(actorEl, null), + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), })); release.entryId = slugify(release.title); - release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60; + release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60; - const poster = query.img('.large_update_thumb', 'src', { origin: channel.url }); - const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1]; + const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url }); + const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1]; release.poster = [ poster.replace(/.jpg$/, '-full.jpg'), poster, ]; - release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [ + release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [ img.replace(/.jpg$/, '-full.jpg'), img, ]); if (trailer) { - release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url); + release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url); } - release.tags = query.cnts('.tour_update_tags a'); + release.tags = query.contents('.tour_update_tags a'); return release; } @@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) { } async function fetchScene(url, channel) { - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - return scrapeScene(res.item, channel); + return scrapeScene(res.context, channel); } return res.status; diff --git a/src/scrapers/analvids.js b/src/scrapers/analvids.js deleted file mode 100644 index dd2670ad..00000000 --- a/src/scrapers/analvids.js +++ /dev/null @@ -1,174 +0,0 @@ -'use strict'; - -const unprint = require('unprint'); - -const slugify = require('../utils/slugify'); - -function extractTitle(originalTitle) { - const titleComponents = originalTitle.split(' '); - // const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes - const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes - const shootId = sceneIdMatch ? sceneIdMatch[0] : null; - const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle; - - return { shootId, title }; -} - -function scrapeAll(scenes, channel) { - return scenes.map(({ query }) => { - const release = {}; - - release.url = query.url('.card-scene__view > a'); - release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1]; - - release.title = query.content('.card-scene__text'); - release.shootId = extractTitle(release.title).shootId; - - release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page - release.duration = query.duration('.label--time:first-child'); - - release.poster = query.img('.card-scene__view img', { attribute: 'data-src' }); - const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`); - - if (caps && release.poster) { - release.caps = caps; - } else if (caps) { - [release.poster, ...release.caps] = caps; - } - - release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' }); - - return release; - }); -} - -async function fetchLatest(channel, page) { - // const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel - // studios as channels - const url = `${channel.url}/latest/${page}`; - - const res = await unprint.get(url, { - selectAll: '.card-scene', - headers: { - Referer: url, - }, - }); - - if (res.ok) { - return scrapeAll(res.context, channel); - } - - return res.status; -} - -function scrapeScene({ query }, url) { - const release = {}; - - release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1]; - - const featuring = query.content('.watch__title .watch__featuring_models'); - - release.title = query.content('.watch__title').replace(featuring, ''); - release.description = query.content('.text-mob-more'); - release.shootId = extractTitle(release.title).shootId; - - release.date = query.date('.bi-calendar3', 'YYYY-MM-DD'); - release.duration = query.duration('.bi-clock'); - - release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({ - name: unprint.query.content(el), - url: unprint.query.url(el, null), - })); - - release.tags = query.contents('.genres-list a[href*="/genre"]'); - - // release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true }); - - release.poster = query.img('.watch__video video', { attribute: 'data-poster' }); - - release.trailer = query.all('.watch__video source').map((el) => ({ - src: unprint.query.video(el, null, { attribute: 'src' }), - quality: unprint.query.number(el, null, { attribute: 'size' }), - })); - - return release; -} - -async function fetchScene(url) { - const res = await unprint.get(url, { - headers: { - Referer: url, - }, - }); - - if (res.ok) { - return scrapeScene(res.context, url); - } - - return res.status; -} - -function scrapeProfile({ query }, url, channel) { - const profile = { url }; - - profile.nationality = query.content('.model__info a[href*="/nationality"]'); - profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td'); - - profile.avatar = query.img('.model__left img'); - - profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel); - - return profile; -} - -async function getActorUrl(actor, channel) { - if (actor.url) { - return actor.url; - } - - const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`; - - const searchRes = await unprint.get(searchUrl, { - headers: { - Referer: actor.url, - }, - }); - - if (!searchRes.ok) { - return searchRes.status; - } - - const result = searchRes.body.terms.find((item) => item.type === 'model'); - - if (result) { - return result.url; - } - - return null; -} - -async function fetchProfile(actor, { channel }) { - const actorUrl = await getActorUrl(actor, channel); - - if (typeof actorUrl !== 'string') { - return actorUrl; - } - - const bioRes = await unprint.get(actorUrl, { - headers: { - Referer: actor.url, - }, - }); - - if (bioRes.ok) { - return scrapeProfile(bioRes.context, actorUrl, channel); - } - - return bioRes.status; -} - -module.exports = { - fetchLatest, - fetchScene, - fetchProfile, -}; diff --git a/src/scrapers/analvids_legacy.js b/src/scrapers/analvids_legacy.js deleted file mode 100755 index efe6d1b2..00000000 --- a/src/scrapers/analvids_legacy.js +++ /dev/null @@ -1,203 +0,0 @@ -'use strict'; - -const { JSDOM } = require('jsdom'); -const cheerio = require('cheerio'); -const moment = require('moment'); - -const http = require('../utils/http'); -const slugify = require('../utils/slugify'); - -function extractTitle(originalTitle) { - const titleComponents = originalTitle.split(' '); - // const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes - const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes - const shootId = sceneIdMatch ? sceneIdMatch[0] : null; - const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle; - - return { shootId, title }; -} - -function getPoster(posterElement, sceneId) { - const posterStyle = posterElement.attr('style'); - - if (posterStyle) { - return posterStyle.slice(posterStyle.indexOf('(') + 1, -1); - } - - const posterRange = posterElement.attr('data-casting'); - const posterRangeData = posterRange ? JSON.parse(posterRange) : null; - const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)]; - - if (!posterTimeRange) { - return null; - } - - if (typeof posterTimeRange === 'number') { - // poster time is already a single time value - return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`; - } - - const [max, min] = posterTimeRange.split('-'); - const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min)); - - return `https://analvids.com/casting/${sceneId}/${posterTime}`; -} - -function scrapeAll(html) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const scenesElements = $('.thumbnails > div').toArray(); - - return scenesElements.map((element) => { - const sceneLinkElement = $(element).find('.thumbnail-title a'); - const url = sceneLinkElement.attr('href'); - - const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping - const { shootId, title } = extractTitle(originalTitle); - const entryId = new URL(url).pathname.split('/')[2]; - - const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate(); - - const sceneId = $(element).attr('data-content'); - const posterElement = $(element).find('.thumbnail-avatar'); - - const poster = getPoster(posterElement, sceneId); - - return { - url, - shootId, - entryId, - title, - date, - poster, - }; - }); -} - -async function scrapeScene(html, url, site, useGallery) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const playerObject = $('script:contains("new WatchPage")').html(); - const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1); - const data = playerData && JSON.parse(playerData); - - const release = { url }; - - const originalTitle = $('h1.watchpage-title').text().trim(); - const { shootId, title } = extractTitle(originalTitle); - - release.shootId = shootId; - release.entryId = new URL(url).pathname.split('/')[2]; - - release.title = title; - release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate(); - - const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray(); - - release.description = $('meta[name="description"]')?.attr('content')?.trim() - || (descriptionElement && $(descriptionElement).find('dd').text().trim()); - - release.actors = $(actorsElement) - .find('a[href*="com/model"]') - .map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - - release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds(); - release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); - - const photos = useGallery - ? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray() - : $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray(); - - release.photos = photos.map((source) => { - // source without parameters sometimes serves larger preview photo - const { origin, pathname } = new URL(source); - - return `${origin}${pathname}`; - }); - - const posterStyle = $('#player').attr('style'); - const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1); - - release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback - - if (data) { - const qualityMap = { - web: 240, - vga: 480, - hd: 720, - '1080p': 1080, - }; - - release.trailer = data.clip.qualities.map((trailer) => ({ - src: trailer.src, - type: trailer.type, - quality: qualityMap[trailer.quality] || trailer.quality, - })); - } - - const studioName = $('.watchpage-studioname').first().text().trim(); - release.studio = slugify(studioName, '', { removePunctuation: true }); - - return release; -} - -async function scrapeProfile(html, _url, actorName) { - const { document } = new JSDOM(html).window; - - const profile = { - name: actorName, - }; - - const avatarEl = document.querySelector('.model--avatar img[src^="http"]'); - const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':')); - - const bio = entries - .filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry) - .reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {}); - - profile.birthPlace = bio.Nationality; - - if (bio.Age) profile.age = bio.Age; - if (avatarEl) profile.avatar = avatarEl.src; - - profile.releases = scrapeAll(html); - - return profile; -} - -async function fetchLatest(site, page = 1) { - const res = await http.get(`${site.url}/new-videos/${page}`); - - return scrapeAll(res.body.toString(), site); -} - -async function fetchScene(url, site) { - const useGallery = true; - - // TODO: fall back on screenshots when gallery is not available - const res = useGallery - ? await http.get(`${url}/gallery#gallery`) - : await http.get(`${url}/screenshots#screenshots`); - - return scrapeScene(res.body.toString(), url, site, useGallery); -} - -async function fetchProfile({ name: actorName }) { - const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`); - const data = res.body; - - const result = data.terms.find((item) => item.type === 'model'); - - if (result) { - const bioRes = await http.get(result.url); - const html = bioRes.body.toString(); - - return scrapeProfile(html, result.url, actorName); - } - - return null; -} - -module.exports = { - fetchLatest, - fetchProfile, - fetchScene, -}; diff --git a/src/scrapers/assylum.js b/src/scrapers/assylum.js index 01ca47c7..6abea372 100755 --- a/src/scrapers/assylum.js +++ b/src/scrapers/assylum.js @@ -1,6 +1,9 @@ 'use strict'; -const { get, geta, ctxa } = require('../utils/q'); +const unprint = require('unprint'); + +// paths are relative, read raw attributes instead of resolved URLs +const raw = { forceGetAttribute: true }; function extractActors(actorString) { return actorString @@ -20,48 +23,48 @@ function matchActors(actorString, models) { } function scrapeLatest(scenes, site, models) { - return scenes.map(({ qu }) => { + return scenes.map(({ query }) => { const release = {}; - const pathname = qu.url('a.itemimg').slice(1); + const pathname = query.attribute('a.itemimg', 'href', raw).slice(1); [release.entryId] = pathname.split('/').slice(-1); release.url = `${site.url}${pathname}`; - release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true); - release.description = qu.q('.mas_longdescription', true); - release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/); + release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a'); + release.description = query.content('.mas_longdescription'); + release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ }); - const actorString = qu.q('.mas_description', true); + const actorString = query.content('.mas_description'); const actors = matchActors(actorString, models); if (actors.length > 0) release.actors = actors; else release.actors = extractActors(actorString); - const posterPath = qu.img('.itemimg img'); + const posterPath = query.attribute('.itemimg img', 'src', raw); release.poster = `${site.url}/${posterPath}`; return release; }); } -function scrapeScene({ html, qu }, url, site, include, models) { +function scrapeScene({ html, query }, url, site, include, models) { const release = { url }; [release.entryId] = url.split('/').slice(-1); - release.title = qu.q('.mas_title', true); - release.description = qu.q('.mas_longdescription', true); - release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); + release.title = query.content('.mas_title'); + release.description = query.content('.mas_longdescription'); + release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ }); if (models) { - const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, ''); + const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, ''); const actors = matchActors(actorString, models); if (actors.length > 0) release.actors = actors; else release.actors = extractActors(actorString); } - release.tags = qu.all('.tags a', true); + release.tags = query.contents('.tags a'); - release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`); + release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`); const posterIndex = 'splash:'; const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4); @@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) { return release; } -function extractModels({ el }, site) { - const models = ctxa(el, '.item'); +function extractModels({ element }, site) { + const models = unprint.initAll(element, '.item'); - return models.map(({ qu }) => { + return models.map(({ query }) => { const actor = { gender: 'female' }; - const avatar = qu.q('.itemimg img'); + const avatar = query.element('.itemimg img'); actor.avatar = `${site.url}/${avatar.src}`; actor.name = avatar.alt .split(':').slice(-1)[0] .replace(/xtreme girl|nurse/ig, '') .trim(); - const actorPath = qu.url('.itemimg'); + const actorPath = query.attribute('.itemimg', 'href', raw); actor.url = `${site.url}${actorPath.slice(1)}`; return actor; @@ -96,11 +99,11 @@ function extractModels({ el }, site) { async function fetchModels(site, page = 1, accModels = []) { const url = `${site.url}/?models/${page}`; - const res = await get(url); + const res = await unprint.get(url); if (res.ok) { - const models = extractModels(res.item, site); - const nextPage = res.item.qa('.pagenumbers', true) + const models = extractModels(res.context, site); + const nextPage = res.context.query.contents('.pagenumbers') .map((pageX) => Number(pageX)) .filter(Boolean) // remove << and >> .includes(page + 1); @@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) { async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) { const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`; - const res = await geta(url, '.item'); + const res = await unprint.get(url, { selectAll: '.item' }); - return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status; + return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status; } async function fetchScene(url, site, release, include, beforeFetchLatest) { const models = beforeFetchLatest || await fetchModels(site); - const res = await get(url); + const res = await unprint.get(url); - return res.ok ? scrapeScene(res.item, url, site, models) : res.status; + return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status; } module.exports = { diff --git a/src/scrapers/badoink.js b/src/scrapers/badoink.js index 373b7bea..2c1ea6c6 100755 --- a/src/scrapers/badoink.js +++ b/src/scrapers/badoink.js @@ -1,6 +1,7 @@ 'use strict'; -const qu = require('../utils/qu'); +const unprint = require('unprint'); + const slugify = require('../utils/slugify'); const { convert } = require('../utils/convert'); @@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) { return scenes.map(({ query }) => { const release = {}; - release.url = query.url('a', 'href', { origin: channel.url }); - release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1]; + release.url = query.url('a', { origin: channel.url }); + release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1]; - release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title'); - release.description = query.cnt('.video-card-description'); + release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title'); + release.description = query.content('.video-card-description'); - release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY'); - release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60; + release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY'); + release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60; release.actors = query.all('.video-card-details--cast a').map((el) => ({ - name: qu.query.cnt(el), - url: qu.query.url(el, null, 'href', { origin: channel.url }), + name: unprint.query.content(el), + url: unprint.query.url(el, null, { origin: channel.url }), })); - release.tags = query.cnts('.video-card-tags a'); + release.tags = query.contents('.video-card-tags a'); - const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')]; + const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })]; release.poster = getPoster(posterSources); release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null); @@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) { }); } -function scrapeScene({ query }, url, channel) { +function scrapeScene({ query }, { url, entity: channel }) { const release = {}; release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1]; - release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]'); - release.description = query.cnt('.video-description'); + release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']); + release.description = query.content('.video-description'); - release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); - release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60; + release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ }); + release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60; release.actors = query.all('.video-actors a').map((el) => ({ - name: qu.query.cnt(el), - url: qu.query.url(el, null, 'href', { origin: channel.url }), + name: unprint.query.content(el), + url: unprint.query.url(el, null, { origin: channel.url }), })); - release.tags = query.cnts('.video-tags a'); + release.tags = query.contents('.video-tags a'); - const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')]; + const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')]; release.poster = getPoster(posterSources); - const photos = query.imgs('.gallery .gallery-item', 'data-big-image'); - const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text'); + const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' }); + const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' }); const lastPhoto = photos.slice(-1)[0]; const photosHaveIndex = /_\d+\./.test(lastPhoto); @@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) { const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({ ...acc, - [slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'), + [slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'), }), {}); - profile.description = query.cnt('.girl-details-bio'); + profile.description = query.content('.girl-details-bio'); profile.age = bio.age; profile.birthPlace = bio.country; @@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) { profile.hairColor = bio.hair; profile.eyes = bio.eyes; - const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')]; + const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')]; profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either profile.social = query.urls('.girl-details-social-media-list a'); - profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity); + profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity); return profile; } async function fetchLatest(channel, page) { - const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', { - Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter + const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, { + selectAll: '.video-card', + cookies: { + affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter + }, }); if (res.ok) { - return scrapeAll(res.items, channel); + return scrapeAll(res.context, channel); } return res.status; @@ -128,10 +132,10 @@ async function fetchLatest(channel, page) { async function fetchProfile(baseActor, { entity }) { const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`; - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - return scrapeProfile(res.item, url, entity); + return scrapeProfile(res.context, url, entity); } return res.status; diff --git a/src/scrapers/czechav.js b/src/scrapers/czechav.js index 4852b9a8..84033391 100755 --- a/src/scrapers/czechav.js +++ b/src/scrapers/czechav.js @@ -1,29 +1,29 @@ 'use strict'; -const qu = require('../utils/q'); +const unprint = require('unprint'); function scrapeAll(scenes, channel) { return scenes.map(({ query }) => { const release = {}; - release.url = query.url('a', 'href', { origin: channel.url }); + release.url = query.url('a', { origin: channel.url }); release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1]; - release.title = query.cnt('.nice-title'); + release.title = query.content('.nice-title'); release.poster = query.img('img.poster'); - release.photos = query.q('img.poster', 'data-urls')?.split(' '); - release.teaser = query.video('img.poster', 'data-thumbnail'); + release.photos = query.attribute('img.poster', 'data-urls')?.split(' '); + release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' }); if (query.exists('.info')) { release.date = query.date('.updated', 'MM/DD/YYYY'); release.actors = [{ - name: query.cnt('.name'), - age: query.cnt('.age'), + name: query.content('.name'), + age: query.content('.age'), }]; - release.tags = query.cnts('.tags a'); + release.tags = query.contents('.tags a'); } return release; @@ -35,38 +35,38 @@ function scrapeScene({ query }, url) { release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1]; - release.title = query.cnt('.nice-title'); - release.description = query.cnt('.desc-text p'); + release.title = query.content('.nice-title'); + release.description = query.content('.desc-text p'); - release.tags = query.cnts('.tags a'); + release.tags = query.contents('.tags a'); release.poster = query.poster(); release.photos = query.imgs('.thumbnails img, .thumb img'); // some videos may not have unique trailers, but many do - release.trailer = query.video(); + release.trailer = query.video('source'); - release.formats = query.cnts('.download-quality option'); + release.formats = query.contents('.download-quality option'); return release; } async function fetchLatest(channel, page = 1) { const url = `${channel.url}/tour/videos/page-${page}`; - const res = await qu.getAll(url, '.episode-list .episode, .content .girl'); + const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' }); if (res.ok) { - return scrapeAll(res.items, channel); + return scrapeAll(res.context, channel); } return res.status; } async function fetchScene(url, channel) { - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - return scrapeScene(res.item, url, channel); + return scrapeScene(res.context, url, channel); } return res.status; diff --git a/src/scrapers/elevatedx.js b/src/scrapers/elevatedx.js index 7a994e11..e11ba4d7 100755 --- a/src/scrapers/elevatedx.js +++ b/src/scrapers/elevatedx.js @@ -1,8 +1,8 @@ 'use strict'; const format = require('template-format'); +const unprint = require('unprint'); -const qu = require('../utils/q'); const slugify = require('../utils/slugify'); const { convert } = require('../utils/convert'); @@ -10,11 +10,11 @@ function deriveEntryId(release) { if (release.date && release.url) { const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1]; - return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`; + return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`; } if (release.date && release.title) { - return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; + return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`; } return null; @@ -22,7 +22,7 @@ function deriveEntryId(release) { function extractPoster(posterPath, channel, baseRelease) { if (posterPath && !/400.jpg/.test(posterPath)) { - const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url); + const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url); const posterSources = [ poster, // upscaled @@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) { return [baseRelease?.poster || null, []]; } -function getImageWithFallbacks(q, selector, site, el) { +function getImageWithFallbacks(query, selector, site, el) { const sources = el ? [ - q(el, selector, 'src0_3x'), - q(el, selector, 'src0_2x'), - q(el, selector, 'src0_1x'), + unprint.query.attribute(el, selector, 'src0_3x'), + unprint.query.attribute(el, selector, 'src0_2x'), + unprint.query.attribute(el, selector, 'src0_1x'), ] : [ - q(selector, 'src0_3x'), - q(selector, 'src0_2x'), - q(selector, 'src0_1x'), + query.attribute(selector, 'src0_3x'), + query.attribute(selector, 'src0_2x'), + query.attribute(selector, 'src0_1x'), ]; - return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url)); + return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url)); } function scrapeAllClassic(scenes, channel) { @@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) { const release = {}; release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])'); - release.entryId = query.url('.updateThumb img', 'alt'); + release.entryId = query.attribute('.updateThumb img', 'alt'); - release.title = query.cnt('.updateInfo h5 a'); + release.title = query.content('.updateInfo h5 a'); - release.actors = query.cnts('.tour_update_models a'); + release.actors = query.contents('.tour_update_models a'); release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY'); - release.poster = query.img('.updateThumb img'); + release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page - const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1]; + const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1]; if (trailer) { release.trailer = `${channel.url}${trailer}`; @@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) { return scenes.map(({ query }) => { const release = {}; - release.title = query.q('h4 a', 'title') || query.q('h4 a', true); + release.title = query.attribute('h4 a', 'title') || query.content('h4 a'); release.url = query.url('h4 a'); release.date = query.date('.more-info-div', 'MMM D, YYYY'); - release.duration = query.dur('.more-info-div'); + release.duration = query.duration('.more-info-div'); - const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder'); + // raw path, prefixed with media URL below + const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true }); if (posterPath) { const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`; @@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) { ]; } - release.teaser = query.video(); + release.teaser = query.video('source'); // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; release.entryId = deriveEntryId(release); @@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) { }); } -function scrapeSceneClassic({ query, html }, url, channel) { +function scrapeSceneClassic({ query, html }, { entity: channel }) { const release = {}; - release.title = query.q('.updatesBlock h2', true); - release.poster = query.meta('property="og:image"'); + release.title = query.content('.updatesBlock h2'); + release.poster = query.attribute('meta[property="og:image"]', 'content'); release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1]; const trailer = html.match(/src="(.+\.mp4)"/)?.[1]; @@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) { return release; } -function scrapeSceneTubular({ query, html }, entity, url, baseRelease) { +function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) { const release = {}; - release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true); + release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title'); release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description'); - release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/); - release.duration = query.dur('.update-info-row:nth-child(2)'); - release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1); + release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ }); + release.duration = query.duration('.update-info-row:nth-child(2)'); + release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 }); release.actors = query.all('.models-list-thumbs a').map((el) => ({ - name: query.cnt(el, 'span'), - avatar: getImageWithFallbacks(query.q, 'img', entity, el), - url: query.url(el, null), + name: unprint.query.content(el, 'span'), + avatar: getImageWithFallbacks(query, 'img', entity, el), + url: unprint.query.url(el, null), })); - release.tags = query.all('.tags a', true); + release.tags = query.contents('.tags a'); - const posterPath = query.q('.player-thumb img', 'src0_1x'); + const posterPath = query.attribute('.player-thumb img', 'src0_1x'); const trailer = html.match(/ avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images - .map((avatar) => qu.prefixUrl(avatar, entity.url)); + .map((avatar) => unprint.prefixUrl(avatar, entity.url)); if (avatarSources.length) profile.avatar = avatarSources; } if (parameters?.layout === 'classic') { - profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity); + profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity); } if (parameters?.layout === 'tubular') { - profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity); + profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity); } return profile; @@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) { const url = (site.parameters?.latest && format(site.parameters.latest, { page })) || `${site.url}/categories/movies_${page}_d.html`; - const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem'); + const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' }); if (!res.ok) { return res.status; } - return allScraper(res.items, site, preData?.uniqueReleases); + return allScraper(res.context, site, preData?.uniqueReleases); } async function fetchUpcomingClassic(channel) { - const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem'); + const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' }); if (res.ok) { - return scrapeAllClassic(res.items, channel); + return scrapeAllClassic(res.context, channel); } return res.status; @@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) { return null; } - const res = await qu.get(profileUrl); + const res = await unprint.get(profileUrl); - if (res.statusCode === 200) { - return scrapeProfile(res.item, entity, parameters); + if (res.status === 200) { + return scrapeProfile(res.context, entity, parameters); } return null; @@ -290,14 +291,12 @@ module.exports = { fetchProfile, scrapeAll: scrapeAllClassic, scrapeScene: scrapeSceneClassic, - deprecated: true, }, tubular: { fetchLatest: fetchLatestTubular, fetchProfile, scrapeAll: scrapeAllTubular, scrapeScene: scrapeSceneTubular, - deprecated: true, }, getImageWithFallbacks, }; diff --git a/src/scrapers/insex.js b/src/scrapers/insex.js index df4eb6a1..a36cfbc7 100755 --- a/src/scrapers/insex.js +++ b/src/scrapers/insex.js @@ -1,25 +1,25 @@ 'use strict'; -const qu = require('../utils/qu'); -const http = require('../utils/http'); +const unprint = require('unprint'); + const slugify = require('../utils/slugify'); function scrapeLatest(scenes, site) { return scenes.map(({ query }) => { const release = {}; - release.url = query.url('figure a', 'href', { origin: site.parameters.latest }); + release.url = query.url('figure a', { origin: site.parameters.latest }); - release.title = query.cnt('.has-text-weight-bold, .is-size-6'); + release.title = query.content('.has-text-weight-bold, .is-size-6'); release.date = query.date('span.tag', 'YYYY-MM-DD'); - release.actors = query.cnts('a.tag'); + release.actors = query.contents('a.tag'); const cover = query.img('.image img'); release.poster = cover.replace('poster_noplay', 'trailer_noplay'); release.covers = [cover]; - release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`; + release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`; return release; }); @@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) { // if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site); const release = {}; - const titleEl = query.q('.galleryTitleText, .articleTitleText'); + const titleEl = query.element('.galleryTitleText, .articleTitleText'); const [title, ...actors] = titleEl.textContent.split('|'); const date = query.date('.articlePostDateText td', 'MMM D, YYYY'); - const url = query.url(titleEl, 'a'); + const url = unprint.query.attribute(titleEl, 'a', 'href'); [release.entryId] = url.split('/').slice(-2); release.url = `${site.url}${url}`; @@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) { } else { // title should contain date instead, not applicable in brief mode release.title = title.slice(title.indexOf(':') + 1).trim(); - release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY'); + release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY'); } release.actors = actors.map((actor) => actor.trim()); - const description = query.q('.articleCopyText', true); + const description = query.content('.articleCopyText'); if (description) release.description = description.slice(0, description.lastIndexOf('(')); - const duration = query.dur('.articleCopyText a:nth-child(2)'); + const duration = query.duration('.articleCopyText a:nth-child(2)'); if (duration) release.duration = duration; - release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10); + release.likes = query.number('.articlePostDateText td:nth-child(3)'); - const cover = query.img('a img'); + const cover = query.img('a img', { origin: site.url }); release.covers = [[ cover.replace('_thumbnail', ''), @@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) { }); } -async function scrapeScene({ query }, url, channel, parameters, session) { +async function scrapeScene({ query }, channel, include, cookies) { const release = {}; - release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold'); - release.description = query.cnt('.has-background-black-ter > div:nth-child(4)'); + release.title = query.content('.columns div.is-size-5.has-text-weight-bold'); + release.description = query.content('.has-background-black-ter > div:nth-child(4)'); release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD'); - release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]'); - release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span'); + release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]'); + release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span'); - release.poster = query.img('#videoPlayer, #iodvideo', 'poster'); - release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]); + release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' }); + release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]); - release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`; + release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`; - release.trailer = query.video(); + release.trailer = query.video('source'); - if (!release.trailer && parameters.includeTrailers) { - const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session }); + if (!release.trailer && include.includeTrailers) { + // play API relies on the session cookie set by the scene page + const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false }); if (trailerRes.ok) { release.trailer = trailerRes.body; @@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) { return release; } -function scrapeSceneLegacy({ query }, site) { +function scrapeSceneLegacy({ query }, { entity: site }) { const release = {}; - const titleEl = query.q('.articleTitleText'); + const titleEl = query.element('.articleTitleText'); const [title, ...actors] = titleEl.textContent.split('|'); - const url = query.url(titleEl, 'a'); + const url = unprint.query.attribute(titleEl, 'a', 'href'); [release.entryId] = url.split('/').slice(-2); release.url = `${site.url}${url}`; release.title = title.trim(); - release.description = query.q('.articleCopyText', true); + release.description = query.content('.articleCopyText'); release.actors = actors.map((actor) => actor.trim()); release.date = query.date('.articlePostDateText', 'MMMM D, YYYY'); - release.duration = query.dur('.articlePostDateText a:nth-child(2)'); + release.duration = query.duration('.articlePostDateText a:nth-child(2)'); const [cover, ...photos] = query.imgs('img[src*="images"]'); release.covers = [cover]; @@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) { release.poster = query.poster(); - const trailer = query.trailer(); + const trailer = query.video('source'); if (trailer) release.trailer = { src: trailer }; return release; @@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) { async function fetchLatest(site, page = 1) { const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`; - const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' }); + const res = await unprint.get(url, { + selectAll: 'body > .columns .column', + cookies: { consent: 'yes' }, + }); if (res.ok) { - return scrapeLatest(res.items, site); + return scrapeLatest(res.context, site); } return res.status; @@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) { async function fetchLatestLegacy(site, page = 1) { const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`; - const res = await qu.get(url); // JSON containing html as a property + const res = await unprint.get(url); // JSON containing html as a property - if (res.ok) { - return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site); + if (res.ok && res.data?.html) { + return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site); } return res.status; } -async function fetchScene(url, site, baseRelease, parameters) { - const session = http.session(); - const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session }); +async function fetchScene(url, site, baseRelease, include) { + const res = await unprint.get(url, { + cookies: { consent: 'yes' }, + }); if (res.ok) { - return scrapeScene(res.item, url, site, parameters, session); + return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies }); } return res.status; diff --git a/src/scrapers/jesseloadsmonsterfacials.js b/src/scrapers/jesseloadsmonsterfacials.js index cf36d422..b6a8dc56 100755 --- a/src/scrapers/jesseloadsmonsterfacials.js +++ b/src/scrapers/jesseloadsmonsterfacials.js @@ -1,11 +1,14 @@ 'use strict'; -const { get, initAll } = require('../utils/qu'); +const unprint = require('unprint'); + +// paths are relative to /visitors/, read raw attributes instead of resolved URLs +const raw = { forceGetAttribute: true }; function scrapeLatest(scenes, dates, site) { - return scenes.map(({ qu }, index) => { + return scenes.map(({ query }, index) => { const release = {}; - const path = qu.url('a[href*="videos/"]'); + const path = query.attribute('a[href*="videos/"]', 'href', raw); if (path) { if (/\.wmv$/.test(path)) { @@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) { } if (dates && dates[index]) { - release.date = dates[index].qu.date(null, 'MM/DD/YYYY'); + release.date = dates[index].query.date(null, 'MM/DD/YYYY'); } // release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId; release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1] - || qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1]; + || query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1]; - release.description = qu.q('tbody tr:nth-child(3) font', true); + release.description = query.content('tbody tr:nth-child(3) font'); - const infoLine = qu.q('font[color="#663366"]', true); + const infoLine = query.content('font[color="#663366"]'); if (infoLine) { release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null; } - const poster = qu.img('img[src*="photos/"][width="400"]'); + const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw); release.poster = `${site.url}/visitors/${poster}`; - release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`); + release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`); return release; }); } -function scrapeScene({ qu }, url, site) { +function scrapeScene({ query }, url, site) { const release = { url }; const { pathname } = new URL(url); @@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) { return release; } - const actor = qu.q('font[color="#990033"] strong', true); + const actor = query.content('font[color="#990033"] strong'); release.actors = [actor]; - const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]'); - const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]'); + const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw); + const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw); release.trailer = [ - { + hdTrailer && { src: `${site.url}/visitors/videos/${hdTrailer}`, quality: 1080, }, - { + sdTrailer && { src: `${site.url}/visitors/videos/${sdTrailer}`, quality: 270, }, - ]; + ].filter(Boolean); return release; } async function fetchLatest(site, page = 1) { const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`; - const res = await get(url); + const res = await unprint.get(url); if (!res.ok) { return res.status; } - const { el } = res.item; + const { element } = res.context; - const scenes = initAll(el, 'table[width="880"]'); - const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)'); + const scenes = unprint.initAll(element, 'table[width="880"]'); + const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)'); return scrapeLatest(scenes, dates, site); } async function fetchScene(url, site) { - const res = await get(url); + const res = await unprint.get(url); if (res.ok) { - return scrapeScene(res.item, url, site); + return scrapeScene(res.context, url, site); } return res.status; diff --git a/src/scrapers/perfectgonzo.js b/src/scrapers/perfectgonzo.js index 7ddd8b4f..f918e6d0 100755 --- a/src/scrapers/perfectgonzo.js +++ b/src/scrapers/perfectgonzo.js @@ -2,7 +2,7 @@ const blake2 = require('blake2'); -const qu = require('../utils/qu'); +const unprint = require('unprint'); function getHash(identifier) { const hash = blake2.createHash('blake2b', { digestLength: 8 }); @@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) { return scenes.map(({ query }) => { const release = {}; - release.title = query.q('a', 'title'); - release.url = query.url('a', 'href', { origin: site.url }); + release.title = query.attribute('a', 'title'); + release.url = query.url('a', { origin: site.url }); release.date = query.date('.nm-date', 'MM/DD/YYYY'); const slug = new URL(release.url).pathname.split('/')[2]; @@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) { [release.poster, ...release.photos] = query.imgs('.bloc-link img'); - release.tags = query.cnts('.dropdown ul a').slice(1); + release.tags = query.contents('.dropdown ul a').slice(1); release.duration = query.duration('.dropdown p:first-child'); return release; @@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) { async function scrapeScene({ query }, site, url) { const release = { url, site }; - release.title = query.cnt('#movie-header h2'); - release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); + release.title = query.content('#movie-header h2'); + release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ }); - release.description = query.cnt('.container .mg-md'); + release.description = query.content('.container .mg-md'); release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)'); - release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container'))); - release.tags = query.cnts('.tag-container a'); + release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container'))); + release.tags = query.contents('.tag-container a'); - const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)'); + const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)'); if (/4K/.test(uhd)) release.tags = release.tags.concat('4k'); release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src); release.poster = query.poster(); - const trailer = query.trailer(); + const trailer = query.video('source'); if (trailer) release.trailer = { src: trailer }; if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site); @@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) { async function fetchLatest(site, page = 1) { const url = `${site.url}/movies/page-${page}`; - const res = await qu.getAll(url, '#content-main [class^="item"]'); + const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' }); if (res.ok) { - return scrapeLatest(res.items, site); + return scrapeLatest(res.context, site); } return res.status; } async function fetchScene(url, channel) { - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - return scrapeScene(res.item, channel, url); + return scrapeScene(res.context, channel, url); } return res.status; diff --git a/src/scrapers/pinkyxxx.js b/src/scrapers/pinkyxxx.js index 63577fb1..4c95a9e0 100755 --- a/src/scrapers/pinkyxxx.js +++ b/src/scrapers/pinkyxxx.js @@ -1,24 +1,42 @@ 'use strict'; -const qu = require('../utils/qu'); -const http = require('../utils/http'); +const unprint = require('unprint'); + +function parseJson(res) { + // admin-ajax responds with JSON labeled as text/html + if (typeof res.data === 'object') { + return res.data; + } + + try { + return JSON.parse(res.body); + } catch (_error) { + return null; + } +} async function getTrailer(entryId, channel) { - const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', { + const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', { action: 'vidplay', vid: entryId, type: 'promo', }, { - encodeJSON: false, - decodeJSON: true, + form: true, }); - if (!videoLinksRes.ok || !videoLinksRes.body.size1) { + const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes); + + if (!videoLinks?.size1) { return null; } - const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url }); - const videoScriptRes = await http.get(scriptUrl); + const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' }); + + if (!scriptUrl) { + return null; + } + + const videoScriptRes = await unprint.get(scriptUrl, { extract: false }); if (!videoScriptRes.ok) { return null; @@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) { const videoData = videoDataString && JSON.parse(videoDataString); const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1]; - const posterUrlEl = videoScriptRes.document.createElement('span'); - - posterUrlEl.innerHTML = posterUrlString; + const poster = posterUrlString && unprint.init(`${posterUrlString}`)?.query.content('span'); // decode HTML entities return { trailer: videoData?.url || null, - poster: posterUrlEl.textContent || null, + poster: poster || null, }; } @@ -48,7 +64,7 @@ function scrapeAll(scenes) { release.title = scene.title; release.description = scene.description; - release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase + release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase release.poster = scene.poster[0]; release.photos = Object.entries(scene).reduce((acc, [key, data]) => { @@ -63,20 +79,20 @@ function scrapeAll(scenes) { }); } -async function scrapeScene({ query }, channel, baseScene) { +async function scrapeScene({ query }, { entity, baseRelease }) { const release = {}; - release.title = query.cnt('.title'); - release.description = query.cnt('.description'); + release.title = query.content('.title'); + release.description = query.content('.description'); - release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1]; + release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1]; - const { poster, trailer } = await getTrailer(release.entryId, channel) || {}; + const { poster, trailer } = await getTrailer(release.entryId, entity) || {}; - release.poster = baseScene?.poster || poster; - release.photos = poster && baseScene?.poster - ? [poster, ...baseScene.photos] - : baseScene?.photos; + release.poster = baseRelease?.poster || poster; + release.photos = poster && baseRelease?.poster + ? [poster, ...baseRelease.photos] + : baseRelease?.photos; release.trailer = trailer; @@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) { } async function fetchLatest(channel, page) { - const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', { + const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', { action: 'vls', limit: 10, offset: (page - 1) * 10, }, { - encodeJSON: false, - decodeJSON: true, + form: true, }); - if (res.ok) { - return scrapeAll(res.body.listings); + const data = res.ok && parseJson(res); + + if (data?.listings) { + return scrapeAll(data.listings); } - return res.status; -} - -async function fetchScene(url, channel, baseScene) { - const res = await qu.get(url); - - if (res.ok) { - return scrapeScene(res.item, channel, baseScene); - } - - return res.status; + return res.ok ? null : res.status; } module.exports = { fetchLatest, - fetchScene, + scrapeScene, }; diff --git a/src/scrapers/releases.js b/src/scrapers/releases.js index e898b358..6d8a811b 100644 --- a/src/scrapers/releases.js +++ b/src/scrapers/releases.js @@ -39,7 +39,6 @@ const karups = require('./karups'); const kellymadison = require('./kellymadison'); const kink = require('./kink'); const mariskax = require('./mariskax'); -// const analvids = require('./analvids'); const pornbox = require('./pornbox'); const littlecapricedreams = require('./littlecapricedreams'); const loveherfilms = require('./loveherfilms'); @@ -74,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld'); const testedefudelidade = require('./testedefudelidade'); const theflourish = require('./theflourish'); const tokyohot = require('./tokyohot'); -// const topwebmodels = require('./topwebmodels'); const traxxx = require('./traxxx'); const virtualtaboo = require('./virtualtaboo'); const vixen = require('./vixen'); diff --git a/src/scrapers/rickysroom.js b/src/scrapers/rickysroom.js index 28dd8c5a..684f540c 100755 --- a/src/scrapers/rickysroom.js +++ b/src/scrapers/rickysroom.js @@ -1,22 +1,23 @@ 'use strict'; -const qu = require('../utils/q'); +const unprint = require('unprint'); + const { lbsToKg, feetInchesToCm } = require('../utils/convert'); function scrapeScene(data, channel) { const release = {}; release.entryId = data.id; - release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url); + release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url); release.title = data.title; release.description = data.description; - release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss'); - release.duration = qu.durationToSeconds(data.videos_duration); + release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss'); + release.duration = unprint.extractDuration(data.videos_duration); release.actors = data?.models_thumbs.map((model) => ({ name: model.name, - url: qu.prefixUrl(`/models/${model.slug}`, channel.url), + url: unprint.prefixUrl(`/models/${model.slug}`, channel.url), avatar: model.thumb, })) || data.models; @@ -42,13 +43,12 @@ function scrapeScene(data, channel) { async function fetchLatest(channel, page = 1) { const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29 - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - const dataString = res.item.query.html('#__NEXT_DATA__'); - const data = dataString && JSON.parse(dataString); + const data = res.context.query.json('#__NEXT_DATA__'); - if (data.props?.pageProps?.contents?.data) { + if (data?.props?.pageProps?.contents?.data) { const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel)); return scenes; @@ -66,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) { return baseRelease; } - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - const dataString = res.item.query.html('#__NEXT_DATA__'); - const data = dataString && JSON.parse(dataString); + const data = res.context.query.json('#__NEXT_DATA__'); - if (data.props?.pageProps?.content) { + if (data?.props?.pageProps?.content) { return scrapeScene(data.props.pageProps.content, channel); } @@ -86,11 +85,11 @@ function scrapeProfile(data, scenes, entity) { const profile = {}; profile.entryId = data.id; - profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url); + profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url); profile.description = data.Bio || data.bio; profile.birthPlace = data.Born || data.born; - profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD'); + profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD'); profile.measurements = data.Measurements || data.Measurements; profile.height = feetInchesToCm(data.Height || data.height); @@ -108,13 +107,12 @@ function scrapeProfile(data, scenes, entity) { async function fetchProfile({ slug }, entity) { const url = `${entity.url}/models/${slug}`; - const res = await qu.get(url); + const res = await unprint.get(url); if (res.ok) { - const dataString = res.item.query.html('#__NEXT_DATA__'); - const data = dataString && JSON.parse(dataString); + const data = res.context.query.json('#__NEXT_DATA__'); - if (data.props?.pageProps?.model) { + if (data?.props?.pageProps?.model) { return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity); } diff --git a/src/scrapers/teamskeet.js b/src/scrapers/teamskeet.js index cf7e7395..d41b8e2c 100755 --- a/src/scrapers/teamskeet.js +++ b/src/scrapers/teamskeet.js @@ -2,7 +2,6 @@ const unprint = require('unprint'); -const http = require('../utils/http'); const slugify = require('../utils/slugify'); const { convert } = require('../utils/convert'); @@ -99,10 +98,10 @@ async function fetchLatest(channel, page = 1, { parameters }) { ? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}` : `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`; - const res = await http.get(url); + const res = await unprint.get(url); - if (res.ok) { - return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); + if (res.ok && res.data?.hits) { + return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters); } return res.status; diff --git a/src/scrapers/topwebmodels.js b/src/scrapers/topwebmodels.js deleted file mode 100755 index f57ed4fb..00000000 --- a/src/scrapers/topwebmodels.js +++ /dev/null @@ -1,144 +0,0 @@ -'use strict'; - -const mime = require('mime'); - -const qu = require('../utils/qu'); -const http = require('../utils/http'); -const slugify = require('../utils/slugify'); -const { convert } = require('../utils/convert'); - -function scrapeSceneX(scene) { - const release = {}; - - release.entryId = scene.id; - release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`; - - release.title = scene.title; - release.description = scene.description; - - release.duration = qu.durationToSeconds(scene.videos_duration); - release.date = new Date(scene.release_date); - - release.actors = scene.models - .map((actor) => (/&/.test(actor.name) - ? actor.name.split(/\s*&\s*/) - : { - name: actor.name, - gender: actor.gender || null, - avatar: actor.thumb, - url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`, - })) - .flat(); - - release.stars = scene.rating; - release.tags = scene.tags.map((tag) => tag.name); - - if (mime.getType(scene.thumb) === 'image/gif') { - release.teaser = scene.thumb; - } else { - release.poster = scene.thumb; - } - - release.channel = slugify(scene.sites[0]?.name, ''); - - return release; -} - -function scrapeAll(scenes) { - return scenes.map(scrapeSceneX); -} - -async function scrapeProfile(actor, options) { - const profile = {}; - - profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch - profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch - profile.gender = actor.attributes.gender || null; - - profile.height = convert(actor.attributes.height.value, 'cm'); - profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg'); - - const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || []; - - profile.bust = Number(bust); - profile.cup = cup; - profile.waist = Number(waist); - profile.hip = Number(hip); - - profile.ethnicity = actor.attributes.ethnicity.value; - profile.birthPlace = actor.attributes.born.value; - - profile.eyes = actor.attributes.eyes.value; - profile.hairColor = actor.attributes.hair.value.split('/')[0]; - - profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`; - profile.avatar = actor.thumb; - - if (options.includeActorScenes) { - const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } }); - - if (res.ok) { - profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items); - } - } - - return profile; -} - -async function fetchLatest(channel, page) { - const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, { - headers: { - Referer: 'https://tour.topwebmodels.com', - 'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey, - 'x-Requested-With': 'XMLHttpRequest', - }, - }); - - if (res.ok) { - return scrapeAll(res.body.videos.items); - } - - return res.status; -} - -async function fetchScene(url) { - const res = await http.get(url, { extract: { runScripts: 'dangerously' } }); - - if (res.ok) { - return { - ...scrapeSceneX(res.window.__DATA__.data.video), - ...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }), - poster: res.window.__DATA__.data.file_poster, - }; - } - - return res.status; -} - -async function fetchProfile(baseActor, entity, options) { - const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, { - headers: { - Referer: 'https://tour.topwebmodels.com', - 'api-key': entity.parameters?.apiKey, - 'x-Requested-With': 'XMLHttpRequest', - }, - }); - - if (!searchRes.ok) { - return searchRes.status; - } - - const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name)); - - if (actor) { - return scrapeProfile(actor, options); - } - - return null; -} - -module.exports = { - fetchLatest, - fetchScene, - fetchProfile, -}; diff --git a/src/store-releases.js b/src/store-releases.js index dcb08cf6..56963808 100755 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -12,7 +12,6 @@ const slugify = require('./utils/slugify'); const bulkInsert = require('./utils/bulk-insert'); const chunk = require('./utils/chunk'); const redis = require('./redis'); -const { formatDate } = require('./utils/qu'); const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors'); const { associateReleaseTags } = require('./tags'); const { curateEntity } = require('./entities'); @@ -32,7 +31,7 @@ const commonContext = { async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { const slugBase = release.title || (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`) - || (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`) + || (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`) || null; const slug = slugify(slugBase, '-', { diff --git a/src/tools/dogfart.js b/src/tools/dogfart.js deleted file mode 100755 index c91ad4b3..00000000 --- a/src/tools/dogfart.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict'; - -const fs = require('fs').promises; - -const Promise = require('bluebird'); - -const qu = require('../utils/qu'); - -const qualities = { - sm: 360, - med: 480, - big: 720, -}; - -/* -async function scrape() { - const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => { - const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`); - - if (res.ok) { - return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' }); - } - - return []; - }, { concurrency: 1 }); - - const urls = urlsByPage.flat(); - - await fs.writeFile('./dogfart-links', urls.join('\n')); - - console.log(`Saved ${urls.length} URLs to file`); -} - -async function compare() { - const newLinksFile = await fs.readFile('./dogfart-links', 'utf8'); - const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8'); - - const newLinks = newLinksFile.split('\n').filter(Boolean); - const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean)); - - const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`); - - await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n')); - - console.log(getLinks); -} -*/ - -async function scrapeMembers() { - const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => { - const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`); - - if (res.ok) { - return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({ - url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`, - actors: query.contents('a[href*="model.php"]'), - trailer: query.urls('.trailer-link a').map((url) => ({ - src: url, - quality: qualities[url.match(/_([a-z]+).mp4/)[1]], - })), - })); - } - - return []; - }, { concurrency: 1 }); - - const urls = titlesByPage.flat().map((data) => JSON.stringify(data)); - - console.log(urls); - - await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n')); - - console.log(`Saved ${urls.length} URLs to file`); -} - -scrapeMembers(); diff --git a/src/utils/media.js b/src/utils/media.js deleted file mode 100755 index b5355ce1..00000000 --- a/src/utils/media.js +++ /dev/null @@ -1,110 +0,0 @@ -'use strict'; - -const config = require('config'); -const fs = require('fs'); -const fsPromises = require('fs').promises; -const Promise = require('bluebird'); -const blake2 = require('blake2'); -const sharp = require('sharp'); -const { nanoid } = require('nanoid'); -const { PassThrough } = require('stream'); - -const http = require('./http'); - -function getMemoryUsage() { - return process.memoryUsage().rss / (10 ** 6); -} - -let peakMemoryUsage = getMemoryUsage(); - -async function fetchSource(link) { - const id = nanoid(); - - const hasher = new blake2.Hash('blake2b'); - hasher.setEncoding('hex'); - - const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`; - const tempFileStream = fs.createWriteStream(tempFilePath); - const hashStream = new PassThrough(); - - hashStream.on('data', (chunk) => hasher.write(chunk)); - - try { - const res = await http.get(link, null, { - stream: true, - transforms: [hashStream], - destination: tempFileStream, - timeout: 5000, - }); - - if (!res.ok) { - throw new Error(res.status); - } - - hasher.end(); - const hash = hasher.read(); - - const memoryUsage = getMemoryUsage(); - peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage); - - console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`); - - return { - id, - path: tempFilePath, - hash, - }; - } catch (error) { - await fsPromises.unlink(tempFilePath); - - throw error; - } -} - -async function init() { - const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8'); - const links = linksFile.split('\n').filter(Boolean); - - await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true }); - - console.time('thumbs'); - - const files = await Promise.map(links, async (link) => { - try { - return await fetchSource(link); - } catch (error) { - console.log(`Failed to fetch ${link}: ${error.message}`); - return null; - } - }); - - await Promise.map(files.filter(Boolean), async (file) => { - const image = sharp(file.path).jpeg(); - - const [{ width, height }, { size }] = await Promise.all([ - image.metadata(), - fsPromises.stat(file.path), - ]); - - await Promise.all([ - image - .toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`), - image - .resize({ - height: config.media.thumbnailSize, - withoutEnlargement: true, - }) - .toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`), - ]); - - const memoryUsage = getMemoryUsage(); - peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage); - - console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`); - }, { concurrency: 10 }); - - console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`); - console.timeEnd('thumbs'); -} - -init(); diff --git a/src/utils/q.js b/src/utils/q.js deleted file mode 100755 index fc445832..00000000 --- a/src/utils/q.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -const qu = require('./qu'); - -module.exports = qu; diff --git a/src/utils/qu.js b/src/utils/qu.js deleted file mode 100755 index af9d46c4..00000000 --- a/src/utils/qu.js +++ /dev/null @@ -1,682 +0,0 @@ -'use strict'; - -const { JSDOM } = require('jsdom'); -const moment = require('moment'); -const http = require('./http'); -const virtualConsole = require('./virtual-console')(__filename); - -const { window: globalWindow } = new JSDOM('', { virtualConsole }); - -function trim(str) { - if (typeof str !== 'string') { - return str; - } - - return str.trim().replace(/\s+/g, ' '); -} - -function extractDate(dateString, format, match) { - if (!dateString) { - return null; - } - - if (match) { - const dateStamp = trim(dateString).match(match); - - if (dateStamp) { - const dateValue = moment.utc(dateStamp[0], format); - - return dateValue.isValid() ? dateValue.toDate() : null; - } - return null; - } - - const dateValue = moment.utc(trim(dateString), format); - - return dateValue.isValid() ? dateValue.toDate() : null; -} - -function formatDate(dateValue, format, inputFormat) { - if (inputFormat) { - return moment(dateValue, inputFormat).format(format); - } - - return moment(dateValue).format(format); -} - -function durationToSeconds(durationString, match) { - const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/); - - if (durationMatch) { - const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3); - - return moment.duration(segments.join(':')).asSeconds(); - } - - return null; -} - -function prefixUrl(urlValue, origin, protocol = 'https') { - if (!urlValue) { - return null; - } - - if (/^http/.test(urlValue)) { - return urlValue; - } - - if (protocol && /^\/\//.test(urlValue)) { - return `${protocol}:${urlValue}`; - } - - if (origin && /^\//.test(urlValue)) { - return `${origin}${urlValue}`; - } - - if (origin && /^\.\//.test(urlValue)) { - return `${origin}${urlValue.slice(1)}`; - } - - if (origin) { - return `${origin}/${urlValue}`; - } - - return urlValue; -} - -function iterateXPathResult(iterator, results = []) { - const element = iterator.iterateNext(); - - if (element) { - return iterateXPathResult(iterator, [...results, element]); - } - - return results; -} - -function getElements(context, selector, first = false) { - if (!selector) { - return context; - } - - if (/^\/\//.test(selector)) { - // XPath selector - const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null); - - if (first) { - return iterator.iterateNext(); - } - - return iterateXPathResult(iterator); - } - - if (first) { - return context.querySelector(selector); - } - - return Array.from(context.querySelectorAll(selector)); -} - -function q(context, selectors, attrArg, applyTrim = true) { - if (!selectors && context.nodeName === '#document') { - return null; - } - - const attr = attrArg === true ? 'textContent' : attrArg; - const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null); - - if (!element) { - return null; - } - - if (attr) { - const value = element[attr] || element.getAttribute(attr); - - return applyTrim && typeof value === 'string' ? trim(value) : value; - } - - return element; -} - -function all(context, selectors, attrArg, applyTrim = true) { - const attr = attrArg === true ? 'textContent' : attrArg; - const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null); - - if (!Array.isArray(elements)) { - return []; - } - - if (attr) { - return elements.map((el) => q(el, null, attr, applyTrim)); - } - - return elements; -} - -function exists(context, selector) { - return !!q(context, selector); -} - -function count(context, selector) { - return all(context, selector)?.length || 0; -} - -function content(context, selector, applyTrim = true) { - return q(context, selector, 'textContent', applyTrim); -} - -function contents(context, selector, applyTrim) { - return all(context, selector, 'textContent', applyTrim); -} - -function html(context, selector) { - const el = q(context, selector, null, true); - - return el && el.innerHTML; -} - -function htmls(context, selector) { - const els = all(context, selector, null, true); - - return els.map((el) => el.innerHTML); -} - -function execute(context, selector = 'script') { - const scripts = htmls(context, selector); - const originalGlobal = Object.fromEntries(Object.entries(global)); - - const errors = scripts?.reduce((accErrors, script) => { - try { - Function(script)(); /* eslint-disable-line no-new-func */ - - return accErrors; - } catch (error) { - // the script failed - return [...accErrors, error]; - } - }, []); - - const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => { - if (originalGlobal[key] !== value) { - delete global[key]; - return true; - } - - return false; - })); - - return { - ...data, - errors, - }; -} - -function json(context, selector) { - const el = q(context, selector, null, true); - - try { - return JSON.parse(el?.innerHTML); - } catch (error) { - return null; - } -} - -function jsons(context, selector) { - const els = all(context, selector, null, true); - - return els.map((el) => { - try { - return JSON.parse(el?.innerHTML); - } catch (error) { - return null; - } - }); -} - -function texts(context, selector, applyTrim = true, filter = true) { - const el = q(context, selector, null, applyTrim); - if (!el) return null; - - const nodes = Array.from(el.childNodes) - .filter((node) => node.nodeName === '#text') - .map((node) => (applyTrim ? trim(node.textContent) : node.textContent)); - - return filter ? nodes.filter(Boolean) : nodes; -} - -function text(context, selector, applyTrim = true) { - const nodes = texts(context, selector, applyTrim, true); - if (!nodes) return null; - - const textValue = nodes.join(' '); - - return applyTrim ? trim(textValue) : textValue; -} - -function removeStyleFunctionSpaces(el) { - // jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png ) - el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`)); -} - -function style(context, selector, styleAttr) { - const el = q(context, selector); - - if (el?.hasAttribute('style')) { - removeStyleFunctionSpaces(el); - - return styleAttr ? el.style[styleAttr] : el.style; - } - - return null; -} - -function styles(context, selector, styleAttr) { - const elStyles = Array.from(context.querySelectorAll(selector), (el) => { - removeStyleFunctionSpaces(el); - - return styleAttr ? el.style[styleAttr] : el.style; - }); - - return elStyles; -} - -function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) { - const value = q(context, selector, attr); - - if (value && match) { - return Number(value.match(match)?.[matchIndex]); - } - - if (value) { - return Number(value); - } - - return null; -} - -function meta(context, selector, attrArg = 'content', applyTrim = true) { - if (/meta\[.*\]/.test(selector)) { - return q(context, selector, attrArg, applyTrim); - } - - return q(context, `meta[${selector}]`, attrArg, applyTrim); -} - -function date(context, selector, format, match, attr = 'textContent') { - const dateString = q(context, selector, attr, true); - - if (!dateString) return null; - - return extractDate(dateString, format, match); -} - -function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') { - const timeString = q(context, selector, attr, 'textContent'); - - if (!timeString) { - return null; - } - - const timeMatch = timeString.match(match); - - if (timeMatch) { - const [n, period] = timeMatch.slice(1); - const thenDate = moment.utc().subtract(Number(n), period); - - return { - date: thenDate.toDate(), - precision: period.replace(/s$/, ''), - }; - } - - return null; -} - -function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) { - const imageEl = (attr && q(context, selector, attr)) - || q(context, selector, 'data-src') - || q(context, selector, 'src'); - - return prefixUrl(imageEl, origin, protocol); -} - -function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) { - const attribute = attr - || (q(context, selector, 'data-src') && 'data-src') - || (q(context, selector, 'src') && 'src'); - - const imageEls = all(context, selector, attribute); - - return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol)); -} - -function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) { - const urlEl = q(context, selector, attr); - const prefixedUrl = prefixUrl(urlEl, origin, protocol); - - if (prefixedUrl && object) { - return new URL(prefixedUrl); - } - - return prefixedUrl; -} - -function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) { - const urlEls = all(context, selector, attr); - - return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls; -} - -function sourceSet(context, selector, attr = 'srcset', options = {}) { - const srcset = q(context, selector, attr); - - if (!srcset) { - return null; - } - - const sources = srcset - .split(/\s*,\s*/) - .map((source) => { - const [link, descriptor] = source.split(' '); - - if (link) { - return { - descriptor: descriptor || 'fallback', - url: prefixUrl(link, options.origin, options.protocol), - }; - } - - return null; - }) - .filter(Boolean) - .sort((sourceA, sourceB) => { - if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) { - return -1; - } - - if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) { - return 1; - } - - return 0; - }); - - if (options.includeDescriptor) { - return sources; - } - - return sources.map((source) => source.url); -} - -function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) { - const posterEl = q(context, selector, attr); - - return attr ? prefixUrl(posterEl, origin, protocol) : posterEl; -} - -function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) { - const trailerEl = q(context, selector, attr); - - return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl; -} - -function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) { - const trailerEls = all(context, selector, attr); - - return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls; -} - -function duration(context, selector, match, attr = 'textContent') { - const durationString = q(context, selector, attr); - - if (!durationString) { - return null; - } - - const durationMatch = durationToSeconds(durationString, match); - - if (durationMatch) { - return durationMatch; - } - - const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i); - - if (timestampMatch) { - const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0; - const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0; - const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0; - - return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds); - } - - return null; -} - -const legacyFuncs = { - q, - qa: all, - qall: all, - qd: date, - qdate: date, - qh: html, - qhtml: html, - qi: image, - qimage: image, - qimages: images, - qis: images, - ql: duration, - qlength: duration, - qm: meta, - qmeta: meta, - qp: poster, - qposter: poster, - qs: all, - qt: video, - qtext: text, - qtexts: texts, - qtrailer: video, - qtrailers: videos, - qts: videos, - qtx: text, - qtxs: texts, - qtxt: text, - qtxts: texts, - // qu: url, - qurl: url, - qurls: urls, - qus: urls, -}; - -const quFuncs = { - all, - cnt: content, - cnts: contents, - content, - contents, - count, - date, - dateAgo, - dur: duration, - duration, - el: q, - element: q, - execute, - exec: execute, - exists, - html, - htmls, - image, - images, - img: image, - imgs: images, - json, - jsons, - length: duration, - meta, - num: number, - number, - poster, - q, - sourceSet, - sources: sourceSet, - srcs: sourceSet, - srcset: sourceSet, - style, - styles, - text, - texts, - trailer: video, - url, - urls, - video, - videos, -}; - -function init(context, selector, window) { - if (!context) { - return null; - } - - const element = selector ? context.querySelector(selector) : context; - - if (!element) { - return null; - } - - const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context - .reduce((acc, [key, func]) => ({ - ...acc, - [key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context - ? func(...args) - : func(element, ...args)), - }), {}); - - const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context - .reduce((acc, [key, func]) => ({ - ...acc, - [key]: (...args) => (args[0]?.nodeType === undefined // allow for different context - ? func(element, ...args) - : func(...args)), - }), {}); - - return { - element, - el: element, - html: element.outerHTML || element.body.outerHTML, - text: trim(element.textContent), - ...(window && { - window, - document: window.document, - }), - ...legacyContextFuncs, - qu: quContextFuncs, - query: quContextFuncs, - }; -} - -function initAll(context, selector, window) { - if (Array.isArray(context)) { - return context.map((element) => init(element, null, window)); - } - - return Array.from(context.querySelectorAll(selector)) - .map((element) => init(element, null, window)); -} - -function extract(htmlValue, selector, options) { - const { window } = new JSDOM(htmlValue, { virtualConsole, ...options }); - - return init(window.document, selector, window); -} - -function extractAll(htmlValue, selector, options) { - const { window } = new JSDOM(htmlValue, { virtualConsole, ...options }); - - return initAll(window.document, selector, window); -} - -async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) { - const res = await (method === 'post' - ? http.post(urlValue, body, { ...options, headers }) - : http[method](urlValue, { - ...options, - headers, - parse: true, - })); - - if (res.ok) { - const item = queryAll - ? initAll(res.document, selector, res.window) - : init(res.document, selector, res.window); - - return { - item, - items: all ? item : [item], - res, - ok: true, - status: res.statusCode, - ...res, - }; - } - - return { - item: null, - items: [], - res, - ok: false, - status: res.statusCode, - ...res, - }; -} - -async function get(urlValue, selector, headers, options) { - return request('get', urlValue, null, selector, headers, options, false); -} - -async function post(urlValue, body, selector, headers, options) { - return request('post', urlValue, body, selector, headers, options, false); -} - -async function getAll(urlValue, selector, headers, options) { - return request('get', urlValue, null, selector, headers, options, true); -} - -async function postAll(urlValue, body, selector, headers, options) { - return request('post', urlValue, body, selector, headers, options, true); -} - -function session(headers, options) { - return http.session(headers, options); -} - -module.exports = { - extractDate, - extract, - extractAll, - durationToSeconds, - init, - initAll, - formatDate, - get, - getAll, - http, - fetch: get, - fetchAll: getAll, - context: init, - contextAll: initAll, - ed: extractDate, - ex: extract, - exa: extractAll, - fd: formatDate, - parseDate: extractDate, - ctx: init, - ctxa: initAll, - geta: getAll, - qu: quFuncs, - query: quFuncs, - post, - postAll, - prefixUrl, - session, - ...legacyFuncs, -}; diff --git a/src/utils/timeout.js b/src/utils/timeout.js deleted file mode 100755 index 9f84f64b..00000000 --- a/src/utils/timeout.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -const http = require('./http'); - -const sleep = 5000; -const timeout = 1000; - -async function init() { - try { - const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, { - timeout, - }); - - console.log(res.statusCode); - } catch (error) { - console.log(error); - } -} - -init();