boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Providing duplicate releases in predata. Using duplicates for filtering scenes without channel in Hush scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-23 02:43:10 +02:00
parent 3a5ea3dd9a
commit 278b74e78c
3 changed files with 34 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/scrapers/hush.js
View File

@@ -107,8 +107,6 @@ function scrapeAllT1(scenes, site, accNetworkReleases) {
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
release.entryId = deriveEntryId(release);
console.log(site.name, accNetworkReleases.map(accRelease => accRelease.entryId), release.entryId, accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId));
if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
return null;
@@ -360,7 +358,7 @@ function scrapeProfileTour({ el, qu }, site) {
return profile;
}
async function fetchLatest(site, page = 1, include, { accNetworkReleases }) {
async function fetchLatest(site, page = 1, include, { uniqueReleases, duplicateReleases }) {
const url = (site.parameters?.latest && util.format(site.parameters.latest, page))
|| (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`)
|| `${site.url}/categories/movies_${page}_d.html`;
@@ -368,10 +366,10 @@ async function fetchLatest(site, page = 1, include, { accNetworkReleases }) {
const res = await geta(url, '.modelfeature, .item-video, .updateItem');
if (!res.ok) return res.status;
if (site.parameters?.t1) return scrapeAllT1(res.items, site, accNetworkReleases);
if (site.parameters?.tour) return scrapeAllTour(res.items, site, accNetworkReleases);
if (site.parameters?.t1) return scrapeAllT1(res.items, site, [...uniqueReleases, ...duplicateReleases]);
if (site.parameters?.tour) return scrapeAllTour(res.items, site);
return scrapeAll(res.items, site, accNetworkReleases);
return scrapeAll(res.items, site, uniqueReleases);
}
async function fetchScene(url, site, baseRelease, include, beforeFetchLatest) {