boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

Browse Source
This commit is contained in:
ThePendulum
2020-03-02 03:41:41 +01:00
parent d0d3d150ee
commit e79a6b33fb
17 changed files with 157 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/releases.js
View File

@@ -202,7 +202,7 @@ async function attachStudio(release) {
};
}
async function curateReleaseEntry(release) {
async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
@@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
};
return curatedRelease;
@@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
sites.slug || ' ' ||
networks.name || ' ' ||
networks.slug || ' ' ||
releases.shoot_id || ' ' ||
EXTRACT(YEAR FROM releases.date) || ' ' ||
CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
}
}
async function storeRelease(release) {
async function storeRelease(release, batchId) {
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
@@ -416,7 +419,7 @@ async function storeRelease(release) {
})
.first();
const curatedRelease = await curateReleaseEntry(release);
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) {
return existingRelease;
@@ -453,11 +456,13 @@ async function storeRelease(release) {
}
async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio);
const { id, slug } = await storeRelease(releaseWithStudio, batchId);
return {
id,