DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

This commit is contained in:
ThePendulum 2020-03-02 03:41:41 +01:00
parent d0d3d150ee
commit e79a6b33fb
17 changed files with 157 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
assets/components/header/filter-bar.vue
View File

@ -3,47 +3,80 @@
<span>
<label class="range">
<input
:id="`${_uid}-new`"
:checked="range === 'new'"
:id="`${_uid}-range-latest`"
:checked="range === 'latest'"
type="radio"
class="range-input"
@click="setRange('new')"
@click="setRange('latest')"
>
<label
:for="`${_uid}-new`"
:for="`${_uid}-range-latest`"
class="range-button"
>New</label>
>Latest</label>
</label>
<label class="range">
<input
:id="`${_uid}-upcoming`"
:id="`${_uid}-range-upcoming`"
:checked="range === 'upcoming'"
type="radio"
class="range-input"
@click="setRange('upcoming')"
>
<label
:for="`${_uid}-upcoming`"
:for="`${_uid}-range-upcoming`"
class="range-button"
>Upcoming</label>
</label>
<label class="range">
<input
:id="`${_uid}-all`"
:id="`${_uid}-range-all`"
:checked="range === 'all'"
type="radio"
class="range-input"
@click="setRange('all')"
>
<label
:for="`${_uid}-all`"
:for="`${_uid}-range-all`"
class="range-button"
>All</label>
</label>
</span>
<span>
<label class="batch">
<input
:id="`${_uid}-batch-all`"
:checked="batch === 'all'"
type="radio"
class="range-input"
@click="setBatch('all')"
>
<label
:for="`${_uid}-batch-all`"
class="range-button"
>All</label>
</label>
<label
v-tooltip="'Only show newly added'"
class="batch"
>
<input
:id="`${_uid}-batch-new`"
:checked="batch === 'new'"
type="radio"
class="range-input"
@click="setBatch('new')"
>
<label
:for="`${_uid}-batch-new`"
class="range-button"
>New</label>
</label>
</span>
<span>
<span class="filters-container">
<Icon icon="filter" />
@ -82,6 +115,10 @@ function range(state) {
return state.ui.range;
}
function batch(state) {
return state.ui.batch;
}
async function setFilter(newFilter) {
this.$store.dispatch('setFilter', newFilter);
@ -94,6 +131,12 @@ async function setRange(newRange) {
await this.fetchReleases();
}
async function setBatch(newBatch) {
this.$store.dispatch('setBatch', newBatch);
await this.fetchReleases();
}
export default {
components: {
Filters,
@ -108,11 +151,13 @@ export default {
...mapState({
filter,
range,
batch,
}),
},
methods: {
setFilter,
setRange,
setBatch,
},
};
</script>

18
assets/components/tile/release.vue
View File

@ -44,7 +44,7 @@
v-tooltip.bottom="release.url && `View scene on ${release.site.name}`"
:title="release.url && `View scene on ${release.site.name}`"
:href="release.url"
:class="{ upcoming: isAfter(release.date, new Date()) }"
:class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
target="_blank"
rel="noopener noreferrer"
class="date"
@ -53,7 +53,7 @@
<a
v-else
:href="release.url"
:class="{ upcoming: isAfter(release.date, new Date()) }"
:class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
title="Scene date N/A, showing date added"
target="_blank"
rel="noopener noreferrer"
@ -224,19 +224,21 @@ export default {
text-decoration: none;
}
.date.upcoming:before {
.date {
&.upcoming:before {
content: '';
color: $text-contrast;
background: $primary;
width: .25rem;
width: .5rem;
display: inline-block;
position: absolute;
top: 0;
bottom: 0;
left: -.75rem;
padding: .25rem;
font-size: .8rem;
left: -.5rem;
}
&.new {
font-weight: bold;
}
}
.site {

6
assets/js/actors/actions.js
View File

@ -52,6 +52,7 @@ function initActorActions(store, _router) {
$limit:Int = 1000,
$after:Date = "1900-01-01",
$before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesActorsOrderBy!]
$exclude: [String!]
) {
@ -132,6 +133,9 @@ function initActorActions(store, _router) {
}
}
}
isNew: {
in: $isNew
}
}
},
first: $limit,
@ -167,6 +171,7 @@ function initActorActions(store, _router) {
limit,
after: store.getters.after,
before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
exclude: store.state.ui.filter,
});
@ -183,7 +188,6 @@ function initActorActions(store, _router) {
if (actorSlug) {
return fetchActorBySlug(actorSlug);
}
console.log(genders);
const { actors } = await graphql(`
query Actors(

5
assets/js/fragments.js
View File

@ -125,7 +125,6 @@ const releaseFields = `
date
slug
type
batch
createdAt
url
${releaseActorsFragment}
@ -139,6 +138,7 @@ const releaseFields = `
slug
url
}
isNew
`;
const releasesFragment = `
@ -157,6 +157,9 @@ const releasesFragment = `
}
}
}
isNew: {
in: $isNew
}
},
first: $limit,
orderBy: $orderBy,

7
assets/js/networks/actions.js
View File

@ -12,6 +12,7 @@ function initNetworksActions(store, _router) {
$before:Date = "2100-01-01",
$afterTime:Datetime = "1900-01-01",
$beforeTime:Datetime = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy: [ReleasesOrderBy!]
$exclude: [String!]
) {
@ -88,7 +89,7 @@ function initNetworksActions(store, _router) {
greaterThan: $afterTime,
}
}
],
]
releasesTags: {
none: {
tag: {
@ -98,6 +99,9 @@ function initNetworksActions(store, _router) {
}
}
}
isNew: {
in: $isNew
}
}
) {
${releaseFields}
@ -110,6 +114,7 @@ function initNetworksActions(store, _router) {
before: store.getters.before,
afterTime: store.getters.after,
beforeTime: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter,
});

4
assets/js/releases/actions.js
View File

@ -9,6 +9,7 @@ function initReleasesActions(store, _router) {
$limit:Int = 1000,
$after:Date = "1900-01-01",
$before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesOrderBy!],
$exclude: [String!]
) {
@ -18,6 +19,7 @@ function initReleasesActions(store, _router) {
limit,
after: store.getters.after,
before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter,
});
@ -41,7 +43,7 @@ function initReleasesActions(store, _router) {
slug
date
url
batch
isNew
site {
id
slug

2
assets/js/sites/actions.js
View File

@ -10,6 +10,7 @@ function initSitesActions(store, _router) {
$limit:Int = 100,
$after:Date = "1900-01-01",
$before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy:[ReleasesOrderBy!]
$exclude: [String!]
) {
@ -38,6 +39,7 @@ function initSitesActions(store, _router) {
limit,
after: store.getters.after,
before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
exclude: store.state.ui.filter,
});

5
assets/js/tags/actions.js
View File

@ -12,6 +12,7 @@ function initTagsActions(store, _router) {
$limit:Int = 1000,
$after:Date = "1900-01-01",
$before:Date = "2100-01-01",
$isNew:[Boolean!] = [true,false]
$orderBy: [ReleasesTagsOrderBy!],
$exclude: [String!]
) {
@ -56,6 +57,9 @@ function initTagsActions(store, _router) {
}
}
}
isNew: {
in: $isNew
}
}
},
first: $limit,
@ -72,6 +76,7 @@ function initTagsActions(store, _router) {
limit,
after: store.getters.after,
before: store.getters.before,
isNew: store.getters.isNew,
orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
exclude: store.state.ui.filter,
});

6
assets/js/ui/actions.js
View File

@ -11,9 +11,15 @@ function initUiActions(_store, _router) {
localStorage.setItem('range', range);
}
function setBatch({ commit }, batch) {
commit('setBatch', batch);
localStorage.setItem('batch', batch);
}
return {
setFilter,
setRange,
setBatch,
};
}

7
assets/js/ui/getters.js
View File

@ -1,7 +1,7 @@
import dayjs from 'dayjs';
const dateRanges = {
new: () => ({
latest: () => ({
after: '1900-01-01',
before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
}),
@ -27,8 +27,13 @@ function after(state) {
return dateRanges[state.range]().after;
}
function isNew(state) {
return state.batch === 'new' ? [true] : [true, false];
}
export default {
rangeDates,
before,
after,
isNew,
};

5
assets/js/ui/mutations.js
View File

@ -6,7 +6,12 @@ function setRange(state, range) {
state.range = range;
}
function setBatch(state, batch) {
state.batch = batch;
}
export default {
setFilter,
setRange,
setBatch,
};

4
assets/js/ui/state.js
View File

@ -1,7 +1,9 @@
const storedFilter = localStorage.getItem('filter');
const storedRange = localStorage.getItem('range');
const storedBatch = localStorage.getItem('batch');
export default {
filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
range: storedRange || 'new',
range: storedRange || 'latest',
batch: storedBatch || 'all',
};

21
migrations/20190325001339_releases.js
View File

@ -327,6 +327,13 @@ exports.up = knex => Promise.resolve()
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('batches', (table) => {
table.increments('id', 12);
table.text('comment');
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('releases', (table) => {
table.increments('id', 16);
@ -362,7 +369,14 @@ exports.up = knex => Promise.resolve()
table.boolean('deep');
table.string('deep_url', 1000);
table.string('batch');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
@ -508,6 +522,10 @@ exports.up = knex => Promise.resolve()
url ILIKE ('%' || search || '%')
$$ LANGUAGE SQL STABLE;
CREATE FUNCTION releases_is_new(release releases) RETURNS boolean AS $$
SELECT NOT EXISTS(SELECT true FROM batches WHERE batches.id = release.created_batch_id + 1 LIMIT 1);
$$ LANGUAGE sql STABLE;
COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
`));
@ -527,6 +545,7 @@ exports.down = knex => knex.raw(`
DROP TABLE IF EXISTS releases_teasers CASCADE;
DROP TABLE IF EXISTS releases_tags CASCADE;
DROP TABLE IF EXISTS releases_search CASCADE;
DROP TABLE IF EXISTS batches CASCADE;
DROP TABLE IF EXISTS actors_avatars CASCADE;
DROP TABLE IF EXISTS actors_photos CASCADE;
DROP TABLE IF EXISTS actors_social CASCADE;

2
src/media.js
View File

@ -271,7 +271,7 @@ function groupItems(items) {
}
async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
const presentSources = sources.filter(Boolean);
const presentSources = sources.filter(source => typeof source === 'string' || Array.isArray(source) || (source && source.src));
if (presentSources.length === 0) {
return {};

13
src/releases.js
View File

@ -202,7 +202,7 @@ async function attachStudio(release) {
};
}
async function curateReleaseEntry(release) {
async function curateReleaseEntry(release, batchId, existingRelease) {
const slug = slugify(release.title, {
encode: true,
limit: config.titleSlugLength,
@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: typeof release.deep === 'boolean' ? release.deep : false,
deep_url: release.deepUrl,
updated_batch_id: batchId,
...(!existingRelease && { created_batch_id: batchId }),
};
return curatedRelease;
@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
sites.slug || ' ' ||
networks.name || ' ' ||
networks.slug || ' ' ||
releases.shoot_id || ' ' ||
EXTRACT(YEAR FROM releases.date) || ' ' ||
CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
}
}
async function storeRelease(release) {
async function storeRelease(release, batchId) {
const existingRelease = await knex('releases')
.where({
entry_id: release.entryId,
@ -416,7 +419,7 @@ async function storeRelease(release) {
})
.first();
const curatedRelease = await curateReleaseEntry(release);
const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
if (existingRelease && !argv.redownload) {
return existingRelease;
@ -453,11 +456,13 @@ async function storeRelease(release) {
}
async function storeReleases(releases) {
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseWithStudio = await attachStudio(releaseWithChannelSite);
const { id, slug } = await storeRelease(releaseWithStudio);
const { id, slug } = await storeRelease(releaseWithStudio, batchId);
return {
id,

10
src/scrapers/private.js
View File

@ -84,12 +84,12 @@ async function scrapeScene(html, url, site) {
release.likes = Number($('.content-desc #social-actions #likes').text());
const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src');
release.trailer = {
src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
};
const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src');
if (trailer) release.trailer = { src: trailer };
release.photos = await getPhotos(release.entryId, site);
release.movie = $('a[data-track="FULL MOVIE"]').attr('href');

6
src/web/plugins/releases.js
View File

@ -4,8 +4,14 @@ const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
const schemaExtender = makeExtendSchemaPlugin(_build => ({
typeDefs: gql`
extend type Release {}
`,
resolvers: {
Release: {
async foo(_parent, _args, _context, _info) {
// template
},
},
},
}));