Scraping Perv City updates to network entity.

This commit is contained in:
ThePendulum 2020-07-12 05:25:27 +02:00
parent 9c8cfe3bdb
commit 70c60e93ac
20 changed files with 22 additions and 6 deletions

View File

@ -94,7 +94,7 @@
<a
v-else
:href="`/channel/${release.entity.slug}`"
:href="`/${release.entity.type}/${release.entity.slug}`"
>
<img
:src="`/img/logos/${release.entity.slug}/thumbs/network.png`"

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 166 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 30 KiB

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -4,7 +4,7 @@ const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
function scrapeAll(scenes) {
function scrapeAll(scenes, entity) {
return scenes.map(({ query }) => {
const release = {};
@ -20,6 +20,7 @@ function scrapeAll(scenes) {
release.actors = query.all('.tour_update_models a', true);
release.poster = query.img('.videoPic img');
release.entity = entity;
return release;
});
@ -76,12 +77,23 @@ function scrapeProfile({ query }) {
}
async function fetchLatest(channel, page = 1) {
if (channel.parameters?.siteId) {
const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
const res = await qu.getAll(url, '.videoBlock');
return res.ok ? scrapeAll(res.items, channel) : res.status;
}
return null;
}
async function fetchUpcoming(channel) {
const url = 'https://pervcity.com';
const res = await qu.getAll(url, '.upcoming .videoBlock');
return res.ok ? scrapeAll(res.items, channel.parent) : res.status;
}
async function fetchScene(url, entity) {
const res = await qu.get(url, '.trailerArea');
@ -110,4 +122,5 @@ module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
fetchUpcoming,
};

View File

@ -94,7 +94,10 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false) {
return accReleases;
}
const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity })); // attach entity the release is assigned to when stored
const latestReleasesWithEntity = latestReleases.map(release => ({
...release,
entity: release.entity || entity, // allow override
})); // attach entity the release is assigned to when stored
const uniqueReleases = argv.redownload
? latestReleasesWithEntity