boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Returning duplicate releases from pagination.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-22 04:22:56 +02:00
parent 5f5c48ea05
commit 3a5ea3dd9a
5 changed files with 46 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/scrapers/hush.js
View File

@@ -10,7 +10,7 @@ const { feetInchesToCm } = require('../utils/convert');
async function getChannelRegExp(site) {
if (!['hushpass', 'interracialpass'].includes(site.parent.slug)) return null;
const sites = await knex('sites').where('network_id', site.parent.id);
const sites = await knex('entities').where('parent_id', site.parent.id);
return new RegExp(sites.map(channel => channel.parameters?.match || channel.name).join('|'), 'i');
}
@@ -80,7 +80,7 @@ function scrapeAll(scenes, site) {
});
}
function scrapeAllT1(scenes, site, accSiteReleases) {
function scrapeAllT1(scenes, site, accNetworkReleases) {
return scenes.map(({ qu }) => {
const release = {};
@@ -107,7 +107,9 @@ function scrapeAllT1(scenes, site, accSiteReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
if (site.parameters?.accFilter && accSiteReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
console.log(site.name, accNetworkReleases.map(accRelease => accRelease.entryId), release.entryId, accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId));
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
return null;
}
@@ -187,6 +189,8 @@ function scrapeSceneT1({ html, qu }, site, url, baseRelease, channelRegExp) {
const stars = qu.q('.update-rating', true).match(/\d.\d/)?.[0];
if (stars) release.stars = Number(stars);
console.log(channelRegExp, site);
if (channelRegExp) {
const channel = release.tags.find(tag => channelRegExp.test(tag));
@@ -356,7 +360,7 @@ function scrapeProfileTour({ el, qu }, site) {
return profile;
}
async function fetchLatest(site, page = 1, include, preflight, accSiteReleases) {
async function fetchLatest(site, page = 1, include, { accNetworkReleases }) {
const url = (site.parameters?.latest && util.format(site.parameters.latest, page))
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
@@ -364,13 +368,13 @@ async function fetchLatest(site, page = 1, include, preflight, accSiteReleases)
const res = await geta(url, '.modelfeature, .item-video, .updateItem');
if (!res.ok) return res.status;
if (site.parameters?.t1) return scrapeAllT1(res.items, site, accSiteReleases);
if (site.parameters?.tour) return scrapeAllTour(res.items, site, accSiteReleases);
if (site.parameters?.t1) return scrapeAllT1(res.items, site, accNetworkReleases);
if (site.parameters?.tour) return scrapeAllTour(res.items, site, accNetworkReleases);
return scrapeAll(res.items, site, accSiteReleases);
return scrapeAll(res.items, site, accNetworkReleases);
}
async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
async function fetchScene(url, site, baseRelease, include, beforeFetchLatest) {
const channelRegExp = beforeFetchLatest || await getChannelRegExp(site);
const res = await get(url);