Added content version table. Marked ElevatedX scraper as deprecated, fixed ExploitedCollegeGirls queries.
This commit is contained in:
parent
cab1823f81
commit
6e1c4a9de8
|
@ -0,0 +1,41 @@
|
||||||
|
exports.up = async (knex) => {
|
||||||
|
await knex.schema.createTable('scenes_revisions', (table) => {
|
||||||
|
table.increments('id');
|
||||||
|
|
||||||
|
table.integer('scene_id')
|
||||||
|
.notNullable()
|
||||||
|
.references('id')
|
||||||
|
.inTable('releases');
|
||||||
|
|
||||||
|
table.integer('user_id')
|
||||||
|
.references('id')
|
||||||
|
.inTable('users');
|
||||||
|
|
||||||
|
table.json('data');
|
||||||
|
table.json('deltas');
|
||||||
|
|
||||||
|
table.text('comment');
|
||||||
|
|
||||||
|
table.boolean('approved')
|
||||||
|
.notNullable()
|
||||||
|
.defaultTo(false);
|
||||||
|
|
||||||
|
table.integer('approved_by')
|
||||||
|
.references('id')
|
||||||
|
.inTable('users');
|
||||||
|
|
||||||
|
table.boolean('applied')
|
||||||
|
.notNullable()
|
||||||
|
.defaultTo(false);
|
||||||
|
|
||||||
|
table.datetime('applied_at');
|
||||||
|
|
||||||
|
table.datetime('created_at')
|
||||||
|
.notNullable()
|
||||||
|
.defaultTo(knex.fn.now());
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.down = async (knex) => {
|
||||||
|
await knex.schema.dropTable('scenes_revisions');
|
||||||
|
};
|
|
@ -108,7 +108,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
||||||
release.entryId = deriveEntryId(release);
|
release.entryId = deriveEntryId(release);
|
||||||
|
|
||||||
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
|
if (channel.parameters?.accFilter && accNetworkReleases?.map((accRelease) => accRelease.entryId).includes(release.entryId)) {
|
||||||
// filter out releases that were already scraped from a categorized site, requeryires sequeryential site scraping
|
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -138,10 +138,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
release.title = query.q('.trailer-section-head .section-title, .title-block .section-title', true);
|
release.title = query.q('.trailer-section-head .section-title, .title-block .section-title', true);
|
||||||
release.description = query.text('.row .update-info-block');
|
release.description = query.text('.update-info-block .description');
|
||||||
|
|
||||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||||
|
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
|
||||||
|
|
||||||
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||||
name: query.cnt(el, 'span'),
|
name: query.cnt(el, 'span'),
|
||||||
|
@ -289,12 +290,14 @@ module.exports = {
|
||||||
fetchProfile,
|
fetchProfile,
|
||||||
scrapeAll: scrapeAllClassic,
|
scrapeAll: scrapeAllClassic,
|
||||||
scrapeScene: scrapeSceneClassic,
|
scrapeScene: scrapeSceneClassic,
|
||||||
|
deprecated: true,
|
||||||
},
|
},
|
||||||
tubular: {
|
tubular: {
|
||||||
fetchLatest: fetchLatestTubular,
|
fetchLatest: fetchLatestTubular,
|
||||||
fetchProfile,
|
fetchProfile,
|
||||||
scrapeAll: scrapeAllTubular,
|
scrapeAll: scrapeAllTubular,
|
||||||
scrapeScene: scrapeSceneTubular,
|
scrapeScene: scrapeSceneTubular,
|
||||||
|
deprecated: true,
|
||||||
},
|
},
|
||||||
getImageWithFallbacks,
|
getImageWithFallbacks,
|
||||||
};
|
};
|
||||||
|
|
|
@ -276,11 +276,11 @@ function styles(context, selector, styleAttr) {
|
||||||
return elStyles;
|
return elStyles;
|
||||||
}
|
}
|
||||||
|
|
||||||
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent') {
|
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
|
||||||
const value = q(context, selector, attr);
|
const value = q(context, selector, attr);
|
||||||
|
|
||||||
if (value && match) {
|
if (value && match) {
|
||||||
return Number(value.match(match)?.[0]);
|
return Number(value.match(match)?.[matchIndex]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (value) {
|
if (value) {
|
||||||
|
|
Loading…
Reference in New Issue