DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Compare commits

base: DebaucheryLibrarian:0ad64ef5f4907f23099837ab5b949ffe7402b7c2
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental
..
compare: DebaucheryLibrarian:d93670842bdd7b137e9d15054ca62239a139db5e
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental

No commits in common. "0ad64ef5f4907f23099837ab5b949ffe7402b7c2" and "d93670842bdd7b137e9d15054ca62239a139db5e" have entirely different histories.
0ad64ef5f4 ... d93670842b

5 changed files with 43 additions and 53 deletions
Show Stats Expand all files Collapse all files

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.204.1",
"version": "1.204.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.204.1",
"version": "1.204.0",
"license": "ISC",
"dependencies": {
"@casl/ability": "^5.2.2",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.204.1",
"version": "1.204.0",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {

48
src/app.js
View File

@ -34,57 +34,47 @@ function logActive() {
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}
async function snapshotMemory() {
const profile = await inspector.heap.takeSnapshot();
const filepath = `traxxx_snapshot_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
/*
function monitorMemory() {
logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
logger.info(`Start heap snapshots, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap dump to ${filepath}`);
if (!done) {
setTimeout(() => monitorMemory(), 10000);
}
}
*/
async function stopMemorySample(snapshotTriggers) {
async function stopMemorySample() {
const profile = await inspector.heap.stopSampling();
const filepath = `traxxx_sample_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
const usage = process.memoryUsage.rss() / 1000000;
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap sample to ${filepath}`);
if (usage > snapshotTriggers[0]) {
await snapshotMemory();
return snapshotTriggers.slice(1);
}
return snapshotTriggers;
}
async function startMemorySample(snapshotTriggers = []) {
async function startMemorySample() {
await inspector.heap.enable();
await inspector.heap.startSampling();
const usage = process.memoryUsage.rss() / 1000000;
// monitorMemory();
logger.info(`Start heap sampling, memory usage: ${usage} MB`);
logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
setTimeout(async () => {
const newSnapshotTriggers = await stopMemorySample(snapshotTriggers);
await stopMemorySample();
if (!done) {
await startMemorySample(newSnapshotTriggers);
await startMemorySample();
}
}, 30000);
}
async function init() {
try {
if (argv.sampleMemory) {
await startMemorySample([1000]);
if (argv.memory) {
await startMemorySample();
}
if (argv.logActive) {
@ -168,8 +158,6 @@ async function init() {
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const storedScenes = argv.save ? await storeScenes(deepScenes) : [];
const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
@ -183,9 +171,9 @@ async function init() {
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedMovieScenes = await storeScenes(deepMovieScenes);
const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
await associateMovieScenes(storedMovies, storedScenes);
}
} catch (error) {
logger.error(error);

24
src/deep.js
View File

@ -54,12 +54,12 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
.filter(Boolean);
}
async function fetchScene(scraper, url, entity, baseRelease, options, type = 'scene') {
if ((type === 'scene' && scraper.fetchScene) || (type === 'movie' && scraper.fetchMovie)) {
return scraper[type === 'movie' ? 'fetchMovie' : 'fetchScene'](baseRelease.url, entity, baseRelease, options, null);
async function fetchScene(scraper, url, entity, baseRelease, options) {
if (scraper.fetchScene) {
return scraper.fetchScene(baseRelease.url, entity, baseRelease, options, null);
}
if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
if (scraper.scrapeScene) {
const session = qu.session();
const res = await qu.get(url, null, null, {
@ -70,7 +70,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
const cookie = await session._sessionOptions.cookieJar.get(url);
if (res.ok) {
return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
return scraper.scrapeScene(res.item, url, entity, baseRelease, options, {
session,
headers: res.headers,
cookieJar: session._sessionOptions.cookieJar,
@ -84,10 +84,6 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
return null;
}
function fetchMovie(scraper, url, entity, baseRelease, options) {
return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
}
async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
const entity = baseRelease.entity || entitiesBySlug[urlToSiteSlug(baseRelease.url)];
@ -110,7 +106,7 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
return baseRelease;
}
if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie && !layoutScraper.scrapeMovie)) {
if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
logger.warn(`The '${entity.name}'-scraper cannot scrape individual ${type}s`);
return baseRelease;
}
@ -127,14 +123,12 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
logger.debug(`Memory usage before: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
const rawScrapedRelease = type === 'scene'
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options)
: await fetchMovie(layoutScraper, baseRelease.url, entity, baseRelease, options);
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
const pathname = new URL(baseRelease.url).pathname.replace(/\//g, '_');
if (rawScrapedRelease) {
delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
}
delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
if (windows.has(pathname)) {
logger.debug(`Closing window for ${pathname}`);

18
src/scrapers/gamma.js
View File

@ -461,7 +461,7 @@ async function fetchMovieTrailer(release) {
return null;
}
async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
async function scrapeMovie({ query, el }, window, url, entity, options) {
const release = {};
const { dataLayer } = query.exec('//script[contains(text(), "dataLayer")]', ['dataLayer']);
@ -495,8 +495,6 @@ async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
release.trailer = await fetchMovieTrailer(release);
}
console.log(release);
return release;
}
@ -753,6 +751,16 @@ async function fetchScene(url, site, baseRelease, options) {
return null;
}
async function fetchMovie(url, channel, baseRelease, options) {
const res = await qu.get(url, null, null);
if (res.ok) {
return scrapeMovie(res.item, res.window, url, channel, options);
}
return res.status;
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
const res = await http.post(apiUrl, {
requests: [
@ -847,6 +855,7 @@ module.exports = {
fetchApiUpcoming: fetchUpcomingApi,
fetchLatest,
fetchLatestApi,
fetchMovie,
fetchProfile,
fetchScene,
fetchSceneApi,
@ -858,13 +867,12 @@ module.exports = {
fetchProfile: fetchApiProfile,
// fetchScene,
fetchScene: fetchSceneApi,
scrapeMovie,
fetchMovie,
},
getPhotos,
scrapeApiProfile,
scrapeApiReleases,
scrapeProfile,
scrapeAll,
scrapeMovie,
scrapeScene,
};