Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

This commit is contained in:
ThePendulum 2020-03-02 03:41:41 +01:00
parent d0d3d150ee
commit e79a6b33fb
17 changed files with 157 additions and 41 deletions

View File

@ -3,47 +3,80 @@
<span> <span>
<label class="range"> <label class="range">
<input <input
:id="`${_uid}-new`" :id="`${_uid}-range-latest`"
:checked="range === 'new'" :checked="range === 'latest'"
type="radio" type="radio"
class="range-input" class="range-input"
@click="setRange('new')" @click="setRange('latest')"
> >
<label <label
:for="`${_uid}-new`" :for="`${_uid}-range-latest`"
class="range-button" class="range-button"
>New</label> >Latest</label>
</label> </label>
<label class="range"> <label class="range">
<input <input
:id="`${_uid}-upcoming`" :id="`${_uid}-range-upcoming`"
:checked="range === 'upcoming'" :checked="range === 'upcoming'"
type="radio" type="radio"
class="range-input" class="range-input"
@click="setRange('upcoming')" @click="setRange('upcoming')"
> >
<label <label
:for="`${_uid}-upcoming`" :for="`${_uid}-range-upcoming`"
class="range-button" class="range-button"
>Upcoming</label> >Upcoming</label>
</label> </label>
<label class="range"> <label class="range">
<input <input
:id="`${_uid}-all`" :id="`${_uid}-range-all`"
:checked="range === 'all'" :checked="range === 'all'"
type="radio" type="radio"
class="range-input" class="range-input"
@click="setRange('all')" @click="setRange('all')"
> >
<label <label
:for="`${_uid}-all`" :for="`${_uid}-range-all`"
class="range-button" class="range-button"
>All</label> >All</label>
</label> </label>
</span> </span>
<span>
<label class="batch">
<input
:id="`${_uid}-batch-all`"
:checked="batch === 'all'"
type="radio"
class="range-input"
@click="setBatch('all')"
>
<label
:for="`${_uid}-batch-all`"
class="range-button"
>All</label>
</label>
<label
v-tooltip="'Only show newly added'"
class="batch"
>
<input
:id="`${_uid}-batch-new`"
:checked="batch === 'new'"
type="radio"
class="range-input"
@click="setBatch('new')"
>
<label
:for="`${_uid}-batch-new`"
class="range-button"
>New</label>
</label>
</span>
<span> <span>
<span class="filters-container"> <span class="filters-container">
<Icon icon="filter" /> <Icon icon="filter" />
@ -82,6 +115,10 @@ function range(state) {
return state.ui.range; return state.ui.range;
} }
function batch(state) {
return state.ui.batch;
}
async function setFilter(newFilter) { async function setFilter(newFilter) {
this.$store.dispatch('setFilter', newFilter); this.$store.dispatch('setFilter', newFilter);
@ -94,6 +131,12 @@ async function setRange(newRange) {
await this.fetchReleases(); await this.fetchReleases();
} }
async function setBatch(newBatch) {
this.$store.dispatch('setBatch', newBatch);
await this.fetchReleases();
}
export default { export default {
components: { components: {
Filters, Filters,
@ -108,11 +151,13 @@ export default {
...mapState({ ...mapState({
filter, filter,
range, range,
batch,
}), }),
}, },
methods: { methods: {
setFilter, setFilter,
setRange, setRange,
setBatch,
}, },
}; };
</script> </script>

View File

@ -44,7 +44,7 @@
v-tooltip.bottom="release.url && `View scene on ${release.site.name}`" v-tooltip.bottom="release.url && `View scene on ${release.site.name}`"
:title="release.url && `View scene on ${release.site.name}`" :title="release.url && `View scene on ${release.site.name}`"
:href="release.url" :href="release.url"
:class="{ upcoming: isAfter(release.date, new Date()) }" :class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
class="date" class="date"
@ -53,7 +53,7 @@
<a <a
v-else v-else
:href="release.url" :href="release.url"
:class="{ upcoming: isAfter(release.date, new Date()) }" :class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
title="Scene date N/A, showing date added" title="Scene date N/A, showing date added"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
@ -224,19 +224,21 @@ export default {
text-decoration: none; text-decoration: none;
} }
.date.upcoming:before { .date {
&.upcoming:before {
content: ''; content: '';
color: $text-contrast;
background: $primary; background: $primary;
width: .25rem; width: .5rem;
display: inline-block; display: inline-block;
position: absolute; position: absolute;
top: 0; top: 0;
bottom: 0; bottom: 0;
left: -.75rem; left: -.5rem;
padding: .25rem; }
font-size: .8rem;
&.new {
font-weight: bold; font-weight: bold;
}
} }
.site { .site {

View File

@ -52,6 +52,7 @@ function initActorActions(store, _router) {
$limit:Int = 1000, $limit:Int = 1000,
$after:Date = "1900-01-01", $after:Date = "1900-01-01",
$before:Date = "2100-01-01", $before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesActorsOrderBy!] $orderBy:[ReleasesActorsOrderBy!]
$exclude: [String!] $exclude: [String!]
) { ) {
@ -132,6 +133,9 @@ function initActorActions(store, _router) {
} }
} }
} }
isNew: {
in: $isNew
}
} }
}, },
first: $limit, first: $limit,
@ -167,6 +171,7 @@ function initActorActions(store, _router) {
limit, limit,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC', orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
@ -183,7 +188,6 @@ function initActorActions(store, _router) {
if (actorSlug) { if (actorSlug) {
return fetchActorBySlug(actorSlug); return fetchActorBySlug(actorSlug);
} }
console.log(genders);
const { actors } = await graphql(` const { actors } = await graphql(`
query Actors( query Actors(

View File

@ -125,7 +125,6 @@ const releaseFields = `
date date
slug slug
type type
batch
createdAt createdAt
url url
${releaseActorsFragment} ${releaseActorsFragment}
@ -139,6 +138,7 @@ const releaseFields = `
slug slug
url url
} }
isNew
`; `;
const releasesFragment = ` const releasesFragment = `
@ -157,6 +157,9 @@ const releasesFragment = `
} }
} }
} }
isNew: {
in: $isNew
}
}, },
first: $limit, first: $limit,
orderBy: $orderBy, orderBy: $orderBy,

View File

@ -12,6 +12,7 @@ function initNetworksActions(store, _router) {
$before:Date = "2100-01-01", $before:Date = "2100-01-01",
$afterTime:Datetime = "1900-01-01", $afterTime:Datetime = "1900-01-01",
$beforeTime:Datetime = "2100-01-01", $beforeTime:Datetime = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy: [ReleasesOrderBy!] $orderBy: [ReleasesOrderBy!]
$exclude: [String!] $exclude: [String!]
) { ) {
@ -88,7 +89,7 @@ function initNetworksActions(store, _router) {
greaterThan: $afterTime, greaterThan: $afterTime,
} }
} }
], ]
releasesTags: { releasesTags: {
none: { none: {
tag: { tag: {
@ -98,6 +99,9 @@ function initNetworksActions(store, _router) {
} }
} }
} }
isNew: {
in: $isNew
}
} }
) { ) {
${releaseFields} ${releaseFields}
@ -110,6 +114,7 @@ function initNetworksActions(store, _router) {
before: store.getters.before, before: store.getters.before,
afterTime: store.getters.after, afterTime: store.getters.after,
beforeTime: store.getters.before, beforeTime: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC', orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });

View File

@ -9,6 +9,7 @@ function initReleasesActions(store, _router) {
$limit:Int = 1000, $limit:Int = 1000,
$after:Date = "1900-01-01", $after:Date = "1900-01-01",
$before:Date = "2100-01-01", $before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesOrderBy!], $orderBy:[ReleasesOrderBy!],
$exclude: [String!] $exclude: [String!]
) { ) {
@ -18,6 +19,7 @@ function initReleasesActions(store, _router) {
limit, limit,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC', orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });
@ -41,7 +43,7 @@ function initReleasesActions(store, _router) {
slug slug
date date
url url
batch isNew
site { site {
id id
slug slug

View File

@ -10,6 +10,7 @@ function initSitesActions(store, _router) {
$limit:Int = 100, $limit:Int = 100,
$after:Date = "1900-01-01", $after:Date = "1900-01-01",
$before:Date = "2100-01-01", $before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesOrderBy!] $orderBy:[ReleasesOrderBy!]
$exclude: [String!] $exclude: [String!]
) { ) {
@ -38,6 +39,7 @@ function initSitesActions(store, _router) {
limit, limit,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC', orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });

View File

@ -12,6 +12,7 @@ function initTagsActions(store, _router) {
$limit:Int = 1000, $limit:Int = 1000,
$after:Date = "1900-01-01", $after:Date = "1900-01-01",
$before:Date = "2100-01-01", $before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy: [ReleasesTagsOrderBy!], $orderBy: [ReleasesTagsOrderBy!],
$exclude: [String!] $exclude: [String!]
) { ) {
@ -56,6 +57,9 @@ function initTagsActions(store, _router) {
} }
} }
} }
isNew: {
in: $isNew
}
} }
}, },
first: $limit, first: $limit,
@ -72,6 +76,7 @@ function initTagsActions(store, _router) {
limit, limit,
after: store.getters.after, after: store.getters.after,
before: store.getters.before, before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC', orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
exclude: store.state.ui.filter, exclude: store.state.ui.filter,
}); });

View File

@ -11,9 +11,15 @@ function initUiActions(_store, _router) {
localStorage.setItem('range', range); localStorage.setItem('range', range);
} }
function setBatch({ commit }, batch) {
commit('setBatch', batch);
localStorage.setItem('batch', batch);
}
return { return {
setFilter, setFilter,
setRange, setRange,
setBatch,
}; };
} }

View File

@ -1,7 +1,7 @@
import dayjs from 'dayjs'; import dayjs from 'dayjs';
const dateRanges = { const dateRanges = {
new: () => ({ latest: () => ({
after: '1900-01-01', after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'), before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
}), }),
@ -27,8 +27,13 @@ function after(state) {
return dateRanges[state.range]().after; return dateRanges[state.range]().after;
} }
function isNew(state) {
return state.batch === 'new' ? [true] : [true, false];
}
export default { export default {
rangeDates, rangeDates,
before, before,
after, after,
isNew,
}; };

View File

@ -6,7 +6,12 @@ function setRange(state, range) {
state.range = range; state.range = range;
} }
function setBatch(state, batch) {
state.batch = batch;
}
export default { export default {
setFilter, setFilter,
setRange, setRange,
setBatch,
}; };

View File

@ -1,7 +1,9 @@
const storedFilter = localStorage.getItem('filter'); const storedFilter = localStorage.getItem('filter');
const storedRange = localStorage.getItem('range'); const storedRange = localStorage.getItem('range');
const storedBatch = localStorage.getItem('batch');
export default { export default {
filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'], filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
range: storedRange || 'new', range: storedRange || 'latest',
batch: storedBatch || 'all',
}; };

View File

@ -327,6 +327,13 @@ exports.up = knex => Promise.resolve()
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
})) }))
.then(() => knex.schema.createTable('batches', (table) => {
table.increments('id', 12);
table.text('comment');
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('releases', (table) => { .then(() => knex.schema.createTable('releases', (table) => {
table.increments('id', 16); table.increments('id', 16);
@ -362,7 +369,14 @@ exports.up = knex => Promise.resolve()
table.boolean('deep'); table.boolean('deep');
table.string('deep_url', 1000); table.string('deep_url', 1000);
table.string('batch'); table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
})) }))
@ -508,6 +522,10 @@ exports.up = knex => Promise.resolve()
url ILIKE ('%' || search || '%') url ILIKE ('%' || search || '%')
$$ LANGUAGE SQL STABLE; $$ LANGUAGE SQL STABLE;
CREATE FUNCTION releases_is_new(release releases) RETURNS boolean AS $$
SELECT NOT EXISTS(SELECT true FROM batches WHERE batches.id = release.created_batch_id + 1 LIMIT 1);
$$ LANGUAGE sql STABLE;
COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many'; COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many'; COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
`)); `));
@ -527,6 +545,7 @@ exports.down = knex => knex.raw(`
DROP TABLE IF EXISTS releases_teasers CASCADE; DROP TABLE IF EXISTS releases_teasers CASCADE;
DROP TABLE IF EXISTS releases_tags CASCADE; DROP TABLE IF EXISTS releases_tags CASCADE;
DROP TABLE IF EXISTS releases_search CASCADE; DROP TABLE IF EXISTS releases_search CASCADE;
DROP TABLE IF EXISTS batches CASCADE;
DROP TABLE IF EXISTS actors_avatars CASCADE; DROP TABLE IF EXISTS actors_avatars CASCADE;
DROP TABLE IF EXISTS actors_photos CASCADE; DROP TABLE IF EXISTS actors_photos CASCADE;
DROP TABLE IF EXISTS actors_social CASCADE; DROP TABLE IF EXISTS actors_social CASCADE;

View File

@ -271,7 +271,7 @@ function groupItems(items) {
} }
async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) { async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
const presentSources = sources.filter(Boolean); const presentSources = sources.filter(source => typeof source === 'string' || Array.isArray(source) || (source && source.src));
if (presentSources.length === 0) { if (presentSources.length === 0) {
return {}; return {};

View File

@ -202,7 +202,7 @@ async function attachStudio(release) {
}; };
} }
async function curateReleaseEntry(release) { async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, { const slug = slugify(release.title, {
encode: true, encode: true,
limit: config.titleSlugLength, limit: config.titleSlugLength,
@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars), // rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false, deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl, deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
}; };
return curatedRelease; return curatedRelease;
@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
sites.slug || ' ' || sites.slug || ' ' ||
networks.name || ' ' || networks.name || ' ' ||
networks.slug || ' ' || networks.slug || ' ' ||
releases.shoot_id || ' ' ||
EXTRACT(YEAR FROM releases.date) || ' ' || EXTRACT(YEAR FROM releases.date) || ' ' ||
CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' || CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' || CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
} }
} }
async function storeRelease(release) { async function storeRelease(release, batchId) {
const existingRelease = await knex('releases') const existingRelease = await knex('releases')
.where({ .where({
entry_id: release.entryId, entry_id: release.entryId,
@ -416,7 +419,7 @@ async function storeRelease(release) {
}) })
.first(); .first();
const curatedRelease = await curateReleaseEntry(release); const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) { if (existingRelease && !argv.redownload) {
return existingRelease; return existingRelease;
@ -453,11 +456,13 @@ async function storeRelease(release) {
} }
async function storeReleases(releases) { async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => { const storedReleases = await Promise.map(releases, async (release) => {
try { try {
const releaseWithChannelSite = await attachChannelSite(release); const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite); const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio); const { id, slug } = await storeRelease(releaseWithStudio, batchId);
return { return {
id, id,

View File

@ -84,12 +84,12 @@ async function scrapeScene(html, url, site) {
release.likes = Number($('.content-desc #social-actions #likes').text()); release.likes = Number($('.content-desc #social-actions #likes').text());
const posterScript = $('script:contains(poster)').html(); const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4); const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
release.poster = $('meta[property="og:image"]').attr('content') || posterLink; release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src');
release.trailer = { const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src');
src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
}; if (trailer) release.trailer = { src: trailer };
release.photos = await getPhotos(release.entryId, site); release.photos = await getPhotos(release.entryId, site);
release.movie = $('a[data-track="FULL MOVIE"]').attr('href'); release.movie = $('a[data-track="FULL MOVIE"]').attr('href');

View File

@ -4,8 +4,14 @@ const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
const schemaExtender = makeExtendSchemaPlugin(_build => ({ const schemaExtender = makeExtendSchemaPlugin(_build => ({
typeDefs: gql` typeDefs: gql`
extend type Release {}
`, `,
resolvers: { resolvers: {
Release: {
async foo(_parent, _args, _context, _info) {
// template
},
},
}, },
})); }));