DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated dependencies. Added periodic memory logger.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-11-20 23:59:15 +01:00
parent a867817dc1
commit 26539b74a5
109 changed files with 10238 additions and 10833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/scrapers/julesjordan.js
View File

@@ -55,7 +55,7 @@ async function getPhotosLegacy(entryId, site, type = 'highres', page = 1) {
// don't add first URL to pages to prevent unnecessary duplicate request
const photos = scrapePhotos(html, type);
const pages = Array.from(new Set($('.page_numbers a').toArray().map(el => $(el).attr('href'))));
const pages = Array.from(new Set($('.page_numbers a').toArray().map((el) => $(el).attr('href'))));
const otherPhotos = pages
? await Promise.map(pages, async (pageX) => {
@@ -84,7 +84,7 @@ async function getPhotos(entryId, site, type = 'highres', page = 1) {
const res = await http.get(albumUrl);
const html = res.body.toString();
const sourceLines = html.split(/\n/).filter(line => line.match(/ptx\["\w+"\]/));
const sourceLines = html.split(/\n/).filter((line) => line.match(/ptx\["\w+"\]/));
const sources = sourceLines.reduce((acc, sourceLine) => {
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
@@ -261,7 +261,7 @@ async function scrapeScene({ html, query }, url, site, include) {
}
if (include.trailer && site.slug !== 'manuelferrara') {
const trailerLines = html.split('\n').filter(line => /movie\["trailer\w*"\]\[/i.test(line));
const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) {
release.trailer = trailerLines.map((trailerLine) => {
@@ -307,7 +307,7 @@ function scrapeMovie({ el, query }, url, site) {
const scenes = scrapeAll(sceneQus, site);
const curatedScenes = scenes
?.map(scene => ({ ...scene, movie }))
?.map((scene) => ({ ...scene, movie }))
.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
movie.date = curatedScenes?.[0].date;
@@ -354,13 +354,13 @@ function scrapeProfile(html, url, actorName, entity) {
avatarEl.getAttribute('src0'),
avatarEl.getAttribute('src'),
]
.filter(avatar => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map(avatar => qu.prefixUrl(avatar, entity.url));
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
.map((avatar) => qu.prefixUrl(avatar, entity.url));
if (avatarSources.length) profile.avatar = avatarSources;
}
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), (el) => el.href);
return profile;
}