Scraping Perv City updates to network entity.

assets/components/releases/release.vue

@@ -94,7 +94,7 @@
 
 					<a
 						v-else
-						:href="`/channel/${release.entity.slug}`"
+						:href="`/${release.entity.type}/${release.entity.slug}`"
 					>
 						<img
 							:src="`/img/logos/${release.entity.slug}/thumbs/network.png`"

src/scrapers/pervcity.js

@@ -4,7 +4,7 @@ const qu = require('../utils/qu');
 const slugify = require('../utils/slugify');
 const { feetInchesToCm, lbsToKg } = require('../utils/convert');
 
-function scrapeAll(scenes) {
+function scrapeAll(scenes, entity) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
@@ -20,6 +20,7 @@ function scrapeAll(scenes) {
 		release.actors = query.all('.tour_update_models a', true);
 
 		release.poster = query.img('.videoPic img');
+		release.entity = entity;
 
 		return release;
 	});
@@ -76,10 +77,21 @@ function scrapeProfile({ query }) {
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
+	if (channel.parameters?.siteId) {
-	const res = await qu.getAll(url, '.videoBlock');
+		const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
+		const res = await qu.getAll(url, '.videoBlock');
 
-	return res.ok ? scrapeAll(res.items, channel) : res.status;
+		return res.ok ? scrapeAll(res.items, channel) : res.status;
+	}
+
+	return null;
+}
+
+async function fetchUpcoming(channel) {
+	const url = 'https://pervcity.com';
+	const res = await qu.getAll(url, '.upcoming .videoBlock');
+
+	return res.ok ? scrapeAll(res.items, channel.parent) : res.status;
 }
 
 async function fetchScene(url, entity) {
@@ -110,4 +122,5 @@ module.exports = {
 	fetchLatest,
 	fetchScene,
 	fetchProfile,
+	fetchUpcoming,
 };

src/updates.js

@@ -94,7 +94,10 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false) {
 			return accReleases;
 		}
 
-		const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity })); // attach entity the release is assigned to when stored
+		const latestReleasesWithEntity = latestReleases.map(release => ({
+			...release,
+			entity: release.entity || entity, // allow override
+		})); // attach entity the release is assigned to when stored
 
 		const uniqueReleases = argv.redownload
 			? latestReleasesWithEntity