Added content version table. Marked ElevatedX scraper as deprecated, fixed ExploitedCollegeGirls queries.

This commit is contained in:
DebaucheryLibrarian 2024-09-08 03:09:44 +02:00
parent cab1823f81
commit 6e1c4a9de8
3 changed files with 48 additions and 4 deletions

View File

@ -0,0 +1,41 @@
exports.up = async (knex) => {
await knex.schema.createTable('scenes_revisions', (table) => {
table.increments('id');
table.integer('scene_id')
.notNullable()
.references('id')
.inTable('releases');
table.integer('user_id')
.references('id')
.inTable('users');
table.json('data');
table.json('deltas');
table.text('comment');
table.boolean('approved')
.notNullable()
.defaultTo(false);
table.integer('approved_by')
.references('id')
.inTable('users');
table.boolean('applied')
.notNullable()
.defaultTo(false);
table.datetime('applied_at');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('scenes_revisions');
};

View File

@ -108,7 +108,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
release.entryId = deriveEntryId(release);
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
return null;
}
@ -138,10 +138,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
const release = {};
release.title = query.q('.trailer-section-head .section-title, .title-block .section-title', true);
release.description = query.text('.row .update-info-block');
release.description = query.text('.update-info-block .description');
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.update-info-row:nth-child(2)');
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
release.actors = query.all('.models-list-thumbs a').map((el) => ({
name: query.cnt(el, 'span'),
@ -289,12 +290,14 @@ module.exports = {
fetchProfile,
scrapeAll: scrapeAllClassic,
scrapeScene: scrapeSceneClassic,
deprecated: true,
},
tubular: {
fetchLatest: fetchLatestTubular,
fetchProfile,
scrapeAll: scrapeAllTubular,
scrapeScene: scrapeSceneTubular,
deprecated: true,
},
getImageWithFallbacks,
};

View File

@ -276,11 +276,11 @@ function styles(context, selector, styleAttr) {
return elStyles;
}
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent') {
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
const value = q(context, selector, attr);
if (value && match) {
return Number(value.match(match)?.[0]);
return Number(value.match(match)?.[matchIndex]);
}
if (value) {