diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 88d0974f..74b9282b 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2681,6 +2681,7 @@ const sites = [ name: 'Booty Annihilation', tags: ['interracial'], parent: 'interracialpass', + hasLogo: false, parameters: { latest: 'https://www.interracialpass.com/t1/categories/BootyAnnihilation_%d_d.html', media: 'https://www.interracialpass.com', diff --git a/src/deep.js b/src/deep.js index 3603ae0e..2b059ff6 100644 --- a/src/deep.js +++ b/src/deep.js @@ -33,9 +33,11 @@ async function findEntities(baseReleases) { )); const entities = await knex('entities') - .select(knex.raw('entities.*, row_to_json(parents) as parent')) + .select(knex.raw('entities.*, row_to_json(parents) as parent, json_agg(children) as children')) .leftJoin('entities as parents', 'parents.id', 'entities.parent_id') + .leftJoin('entities as children', 'children.parent_id', 'entities.id') .whereIn('entities.slug', entitySlugs) + .groupBy('entities.id', 'parents.id') .orderBy('entities.type', 'asc'); // channel entity will overwrite network entity diff --git a/src/entities.js b/src/entities.js index 819798a5..73ff4e41 100644 --- a/src/entities.js +++ b/src/entities.js @@ -86,7 +86,7 @@ async function fetchIncludedEntities() { ) /* select recursive channels as children of networks */ SELECT - entities.*, json_agg(channels) as children + entities.*, json_agg(channels ORDER BY channels.id) as children FROM channels LEFT JOIN diff --git a/src/scrapers/hush.js b/src/scrapers/hush.js index f5e2adfd..41f71c89 100644 --- a/src/scrapers/hush.js +++ b/src/scrapers/hush.js @@ -10,7 +10,7 @@ const { feetInchesToCm } = require('../utils/convert'); async function getChannelRegExp(site) { if (!['hushpass', 'interracialpass'].includes(site.parent.slug)) return null; - const sites = await knex('sites').where('network_id', site.parent.id); + const sites = await knex('entities').where('parent_id', site.parent.id); return new RegExp(sites.map(channel => channel.parameters?.match || channel.name).join('|'), 'i'); } @@ -80,7 +80,7 @@ function scrapeAll(scenes, site) { }); } -function scrapeAllT1(scenes, site, accSiteReleases) { +function scrapeAllT1(scenes, site, accNetworkReleases) { return scenes.map(({ qu }) => { const release = {}; @@ -107,7 +107,9 @@ function scrapeAllT1(scenes, site, accSiteReleases) { // release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1]; release.entryId = deriveEntryId(release); - if (site.parameters?.accFilter && accSiteReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) { + console.log(site.name, accNetworkReleases.map(accRelease => accRelease.entryId), release.entryId, accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)); + + if (site.parameters?.accFilter && accNetworkReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) { // filter out releases that were already scraped from a categorized site, requires sequential site scraping return null; } @@ -187,6 +189,8 @@ function scrapeSceneT1({ html, qu }, site, url, baseRelease, channelRegExp) { const stars = qu.q('.update-rating', true).match(/\d.\d/)?.[0]; if (stars) release.stars = Number(stars); + console.log(channelRegExp, site); + if (channelRegExp) { const channel = release.tags.find(tag => channelRegExp.test(tag)); @@ -356,7 +360,7 @@ function scrapeProfileTour({ el, qu }, site) { return profile; } -async function fetchLatest(site, page = 1, include, preflight, accSiteReleases) { +async function fetchLatest(site, page = 1, include, { accNetworkReleases }) { const url = (site.parameters?.latest && util.format(site.parameters.latest, page)) || (site.parameters?.t1 && `${site.url}/t1/categories/movies_${page}_d.html`) || `${site.url}/categories/movies_${page}_d.html`; @@ -364,13 +368,13 @@ async function fetchLatest(site, page = 1, include, preflight, accSiteReleases) const res = await geta(url, '.modelfeature, .item-video, .updateItem'); if (!res.ok) return res.status; - if (site.parameters?.t1) return scrapeAllT1(res.items, site, accSiteReleases); - if (site.parameters?.tour) return scrapeAllTour(res.items, site, accSiteReleases); + if (site.parameters?.t1) return scrapeAllT1(res.items, site, accNetworkReleases); + if (site.parameters?.tour) return scrapeAllTour(res.items, site, accNetworkReleases); - return scrapeAll(res.items, site, accSiteReleases); + return scrapeAll(res.items, site, accNetworkReleases); } -async function fetchScene(url, site, baseRelease, beforeFetchLatest) { +async function fetchScene(url, site, baseRelease, include, beforeFetchLatest) { const channelRegExp = beforeFetchLatest || await getChannelRegExp(site); const res = await get(url); diff --git a/src/updates.js b/src/updates.js index d1290dd7..e5333aa0 100644 --- a/src/updates.js +++ b/src/updates.js @@ -10,6 +10,8 @@ const include = require('./utils/argv-include')(argv); const scrapers = require('./scrapers/scrapers'); const { fetchIncludedEntities } = require('./entities'); +const emptyReleases = { uniqueReleases: [], duplicateReleases: [] }; + async function filterUniqueReleases(latestReleases, accReleases) { const latestReleaseIdentifiers = latestReleases .map(release => [release.entity.id, release.entryId]); @@ -31,10 +33,9 @@ async function filterUniqueReleases(latestReleases, accReleases) { return acc; }, {}); - const uniqueReleases = latestReleases - .filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]); + const uniqueReleases = latestReleases.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]); - return uniqueReleases; + return { uniqueReleases, duplicateReleases }; } function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) { @@ -66,7 +67,7 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin return false; } -async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) { +async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, acc = emptyReleases, totalReleases = 0) { const releases = upcoming ? await scraper.fetchUpcoming(entity, page, include, preData) : await scraper.fetchLatest(entity, page, include, preData); @@ -74,7 +75,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page = if (!Array.isArray(releases)) { // scraper is unable to fetch the releases and returned a HTTP code or null logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`); - return accReleases; + return acc; } const releasesWithEntity = releases.map(release => ({ @@ -88,20 +89,25 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page = || (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after))) || releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0)); - const uniqueReleases = argv.force - ? limitedReleases - : await filterUniqueReleases(limitedReleases, accReleases); + const { uniqueReleases, duplicateReleases } = argv.force + ? { uniqueReleases: limitedReleases, duplicateReleases: [] } + : await filterUniqueReleases(limitedReleases, acc.uniqueReleases); + + const accReleases = { + uniqueReleases: acc.uniqueReleases.concat(uniqueReleases), + duplicateReleases: acc.duplicateReleases.concat(duplicateReleases), + }; if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) { - return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length); + return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases, totalReleases + releases.length); } - return accReleases.concat(uniqueReleases); + return accReleases; } async function scrapeLatestReleases(scraper, entity, preData) { - if (!scraper.fetchLatest) { - return []; + if (!argv.latest || !scraper.fetchLatest) { + return emptyReleases; } try { @@ -114,12 +120,12 @@ async function scrapeLatestReleases(scraper, entity, preData) { logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); } - return []; + return emptyReleases; } async function scrapeUpcomingReleases(scraper, entity, preData) { - if (!scraper.fetchUpcoming) { - return []; + if (!argv.upcoming || !scraper.fetchUpcoming) { + return emptyReleases; } try { @@ -132,11 +138,11 @@ async function scrapeUpcomingReleases(scraper, entity, preData) { logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); } - return []; + return emptyReleases; } async function scrapeMovies(scraper, entity) { - if (!scraper.fetchMovies) { + if (!argv.movies || !scraper.fetchMovies) { return []; } @@ -152,20 +158,16 @@ async function scrapeMovies(scraper, entity) { async function scrapeChannelReleases(scraper, channelEntity, preData) { const [latestReleases, upcomingReleases] = await Promise.all([ - argv.latest - ? scrapeLatestReleases(scraper, channelEntity, preData) - : [], - argv.upcoming - ? scrapeUpcomingReleases(scraper, channelEntity, preData) - : [], - argv.movies - ? scrapeMovies(scraper, channelEntity, preData) - : [], + scrapeLatestReleases(scraper, channelEntity, preData), + scrapeUpcomingReleases(scraper, channelEntity, preData), + scrapeMovies(scraper, channelEntity, preData), ]); + console.log(latestReleases); + logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`); - return [...latestReleases, ...upcomingReleases]; + return [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases]; } async function scrapeChannel(channelEntity, accNetworkReleases) {