DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Scraping from Cherry Pimps when available. Showing cover in movie tile.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-02 03:44:14 +02:00
parent 767437d9aa
commit b4f0501765
6 changed files with 63 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/scrapers/cherrypimps.js
View File

@@ -31,28 +31,34 @@ function scrapeAll(scenes, site) {
}).filter(Boolean);
}
function scrapeScene({ q, qd, qa }, url, _site, baseRelease) {
function scrapeScene({ query, html }, url, _site, baseRelease) {
const release = { url };
const { pathname } = new URL(url);
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
release.title = q('.trailer-block_title', true);
release.description = q('.info-block:nth-child(3) .text', true);
release.date = qd('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.title = query.q('.trailer-block_title', true);
release.description = query.q('.info-block:nth-child(3) .text', true);
release.date = query.date('.info-block_data .text', 'MMMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
const duration = baseRelease?.duration || Number(q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60;
const duration = baseRelease?.duration || Number(query.q('.info-block_data .text', true).match(/(\d+)\s+min/)?.[1]) * 60;
if (duration) release.duration = duration;
release.actors = qa('.info-block_data a[href*="/models"]', true);
release.tags = qa('.info-block a[href*="/categories"]', true);
release.actors = query.all('.info-block_data a[href*="/models"]', true);
release.tags = query.all('.info-block a[href*="/categories"]', true);
const posterEl = q('.update_thumb');
const poster = posterEl.getAttribute('src0_3x') || posterEl.getAttribute('src0_2x') || posterEl.dataset.src;
const posterEl = query.q('.update_thumb');
const poster = posterEl?.getAttribute('src0_3x') || posterEl?.getAttribute('src0_2x') || posterEl?.dataset.src;
if (poster && baseRelease?.poster) release.photos = [poster];
else if (poster) release.poster = poster;
const trailer = html.match(/video src="(.*?)"/);
if (trailer) {
release.trailer = trailer[1];
}
return release;
}

7
src/store-releases.js
View File

@@ -23,8 +23,6 @@ function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
limit: config.titleSlugLength,
});
console.log(release);
const curatedRelease = {
title: release.title,
entry_id: release.entryId || null,
@@ -47,7 +45,7 @@ function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
if (type === 'scene') {
curatedRelease.shoot_id = release.shootId || null;
curatedRelease.productionDate = Number(release.productionDate) ? release.productionDate : null;
curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null;
curatedRelease.duration = release.duration;
}
@@ -260,10 +258,7 @@ async function storeReleases(releases) {
async function storeMovies(movies) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
console.log(movies);
const curatedMovieEntries = movies.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
console.log(curatedMovieEntries);
const storedMovies = await knex.batchInsert('movies', curatedMovieEntries).returning('*');
const moviesWithId = attachReleaseIds(movies, storedMovies);