DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

Browse Source
This commit is contained in:
ThePendulum
2020-03-02 03:41:41 +01:00
parent d0d3d150ee
commit e79a6b33fb
17 changed files with 157 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/releases.js
View File

@@ -202,7 +202,7 @@ async function attachStudio(release) {
};
}
async function curateReleaseEntry(release) {
async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
@@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
};
return curatedRelease;
@@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
sites.slug || ' ' ||
networks.name || ' ' ||
networks.slug || ' ' ||
releases.shoot_id || ' ' ||
EXTRACT(YEAR FROM releases.date) || ' ' ||
CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
}
}
async function storeRelease(release) {
async function storeRelease(release, batchId) {
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
@@ -416,7 +419,7 @@ async function storeRelease(release) {
})
.first();
const curatedRelease = await curateReleaseEntry(release);
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) {
return existingRelease;
@@ -453,11 +456,13 @@ async function storeRelease(release) {
}
async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio);
const { id, slug } = await storeRelease(releaseWithStudio, batchId);
return {
id,