Added content version table. Marked ElevatedX scraper as deprecated, fixed ExploitedCollegeGirls queries.
This commit is contained in:
parent
cab1823f81
commit
6e1c4a9de8
|
@ -0,0 +1,41 @@
|
|||
exports.up = async (knex) => {
|
||||
await knex.schema.createTable('scenes_revisions', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('scene_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('releases');
|
||||
|
||||
table.integer('user_id')
|
||||
.references('id')
|
||||
.inTable('users');
|
||||
|
||||
table.json('data');
|
||||
table.json('deltas');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.boolean('approved')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.integer('approved_by')
|
||||
.references('id')
|
||||
.inTable('users');
|
||||
|
||||
table.boolean('applied')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.datetime('applied_at');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async (knex) => {
|
||||
await knex.schema.dropTable('scenes_revisions');
|
||||
};
|
|
@ -108,7 +108,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
|||
release.entryId = deriveEntryId(release);
|
||||
|
||||
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
|
||||
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
||||
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -138,10 +138,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
|||
const release = {};
|
||||
|
||||
release.title = query.q('.trailer-section-head .section-title, .title-block .section-title', true);
|
||||
release.description = query.text('.row .update-info-block');
|
||||
release.description = query.text('.update-info-block .description');
|
||||
|
||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
|
||||
|
||||
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||
name: query.cnt(el, 'span'),
|
||||
|
@ -289,12 +290,14 @@ module.exports = {
|
|||
fetchProfile,
|
||||
scrapeAll: scrapeAllClassic,
|
||||
scrapeScene: scrapeSceneClassic,
|
||||
deprecated: true,
|
||||
},
|
||||
tubular: {
|
||||
fetchLatest: fetchLatestTubular,
|
||||
fetchProfile,
|
||||
scrapeAll: scrapeAllTubular,
|
||||
scrapeScene: scrapeSceneTubular,
|
||||
deprecated: true,
|
||||
},
|
||||
getImageWithFallbacks,
|
||||
};
|
||||
|
|
|
@ -276,11 +276,11 @@ function styles(context, selector, styleAttr) {
|
|||
return elStyles;
|
||||
}
|
||||
|
||||
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent') {
|
||||
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
|
||||
const value = q(context, selector, attr);
|
||||
|
||||
if (value && match) {
|
||||
return Number(value.match(match)?.[0]);
|
||||
return Number(value.match(match)?.[matchIndex]);
|
||||
}
|
||||
|
||||
if (value) {
|
||||
|
|
Loading…
Reference in New Issue