Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

assets/components/header/filter-bar.vue

@@ -3,47 +3,80 @@
         <span>
             <label class="range">
                 <input
-                    :id="`${_uid}-new`"
+                    :id="`${_uid}-range-latest`"
-                    :checked="range === 'new'"
+                    :checked="range === 'latest'"
                     type="radio"
                     class="range-input"
-                    @click="setRange('new')"
+                    @click="setRange('latest')"
                 >
                 <label
-                    :for="`${_uid}-new`"
+                    :for="`${_uid}-range-latest`"
                     class="range-button"
-                >New</label>
+                >Latest</label>
             </label>
 
             <label class="range">
                 <input
-                    :id="`${_uid}-upcoming`"
+                    :id="`${_uid}-range-upcoming`"
                     :checked="range === 'upcoming'"
                     type="radio"
                     class="range-input"
                     @click="setRange('upcoming')"
                 >
                 <label
-                    :for="`${_uid}-upcoming`"
+                    :for="`${_uid}-range-upcoming`"
                     class="range-button"
                 >Upcoming</label>
             </label>
 
             <label class="range">
                 <input
-                    :id="`${_uid}-all`"
+                    :id="`${_uid}-range-all`"
                     :checked="range === 'all'"
                     type="radio"
                     class="range-input"
                     @click="setRange('all')"
                 >
                 <label
-                    :for="`${_uid}-all`"
+                    :for="`${_uid}-range-all`"
                     class="range-button"
                 >All</label>
             </label>
         </span>
 
+        <span>
+            <label class="batch">
+                <input
+                    :id="`${_uid}-batch-all`"
+                    :checked="batch === 'all'"
+                    type="radio"
+                    class="range-input"
+                    @click="setBatch('all')"
+                >
+                <label
+                    :for="`${_uid}-batch-all`"
+                    class="range-button"
+                >All</label>
+            </label>
+
+            <label
+                v-tooltip="'Only show newly added'"
+                class="batch"
+            >
+                <input
+                    :id="`${_uid}-batch-new`"
+                    :checked="batch === 'new'"
+                    type="radio"
+                    class="range-input"
+                    @click="setBatch('new')"
+                >
+                <label
+                    :for="`${_uid}-batch-new`"
+                    class="range-button"
+                >New</label>
+            </label>
+        </span>
+
         <span>
             <span class="filters-container">
                 <Icon icon="filter" />
@@ -82,6 +115,10 @@ function range(state) {
     return state.ui.range;
 }
 
+function batch(state) {
+    return state.ui.batch;
+}
+
 async function setFilter(newFilter) {
     this.$store.dispatch('setFilter', newFilter);
 
@@ -94,6 +131,12 @@ async function setRange(newRange) {
     await this.fetchReleases();
 }
 
+async function setBatch(newBatch) {
+    this.$store.dispatch('setBatch', newBatch);
+
+    await this.fetchReleases();
+}
+
 export default {
     components: {
         Filters,
@@ -108,11 +151,13 @@ export default {
         ...mapState({
             filter,
             range,
+            batch,
         }),
     },
     methods: {
         setFilter,
         setRange,
+        setBatch,
     },
 };
 </script>

assets/components/tile/release.vue

@@ -44,7 +44,7 @@
                     v-tooltip.bottom="release.url && `View scene on ${release.site.name}`"
                     :title="release.url && `View scene on ${release.site.name}`"
                     :href="release.url"
-                    :class="{ upcoming: isAfter(release.date, new Date()) }"
+                    :class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
                     target="_blank"
                     rel="noopener noreferrer"
                     class="date"
@@ -53,7 +53,7 @@
                 <a
                     v-else
                     :href="release.url"
-                    :class="{ upcoming: isAfter(release.date, new Date()) }"
+                    :class="{ upcoming: isAfter(release.date, new Date()), new: release.isNew }"
                     title="Scene date N/A, showing date added"
                     target="_blank"
                     rel="noopener noreferrer"
@@ -224,19 +224,21 @@ export default {
     text-decoration: none;
 }
 
-.date.upcoming:before {
+.date {
-    content: '';
+    &.upcoming:before {
-    color: $text-contrast;
+        content: '';
-    background: $primary;
+        background: $primary;
-    width: .25rem;
+        width: .5rem;
-    display: inline-block;
+        display: inline-block;
-    position: absolute;
+        position: absolute;
-    top: 0;
+        top: 0;
-    bottom: 0;
+        bottom: 0;
-    left: -.75rem;
+        left: -.5rem;
-    padding: .25rem;
+    }
-    font-size: .8rem;
+
-    font-weight: bold;
+    &.new {
+        font-weight: bold;
+    }
 }
 
 .site {

assets/js/actors/actions.js

@@ -52,6 +52,7 @@ function initActorActions(store, _router) {
                 $limit:Int = 1000,
                 $after:Date = "1900-01-01",
                 $before:Date = "2100-01-01",
+                $isNew:[Boolean!] = [true,false]
                 $orderBy:[ReleasesActorsOrderBy!]
                 $exclude: [String!]
             ) {
@@ -132,6 +133,9 @@ function initActorActions(store, _router) {
                                         }
                                     }
                                 }
+                                isNew: {
+                                    in: $isNew
+                                }
                             }
                         },
                         first: $limit,
@@ -167,6 +171,7 @@ function initActorActions(store, _router) {
             limit,
             after: store.getters.after,
             before: store.getters.before,
+            isNew: store.getters.isNew,
             orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
             exclude: store.state.ui.filter,
         });
@@ -183,7 +188,6 @@ function initActorActions(store, _router) {
         if (actorSlug) {
             return fetchActorBySlug(actorSlug);
         }
-        console.log(genders);
 
         const { actors } = await graphql(`
             query Actors(

assets/js/fragments.js

@@ -125,7 +125,6 @@ const releaseFields = `
     date
     slug
     type
-    batch
     createdAt
     url
     ${releaseActorsFragment}
@@ -139,6 +138,7 @@ const releaseFields = `
         slug
         url
     }
+    isNew
 `;
 
 const releasesFragment = `
@@ -157,6 +157,9 @@ const releasesFragment = `
                     }
                 }
             }
+            isNew: {
+                in: $isNew
+            }
         },
         first: $limit,
         orderBy: $orderBy,

assets/js/networks/actions.js

@@ -12,6 +12,7 @@ function initNetworksActions(store, _router) {
                 $before:Date = "2100-01-01",
                 $afterTime:Datetime = "1900-01-01",
                 $beforeTime:Datetime = "2100-01-01",
+                $isNew:[Boolean!] = [true,false]
                 $orderBy: [ReleasesOrderBy!]
                 $exclude: [String!]
             ) {
@@ -88,7 +89,7 @@ function initNetworksActions(store, _router) {
                                     greaterThan: $afterTime,
                                 }
                             }
-                        ],
+                        ]
                         releasesTags: {
                             none: {
                                 tag: {
@@ -98,6 +99,9 @@ function initNetworksActions(store, _router) {
                                 }
                             }
                         }
+                        isNew: {
+                            in: $isNew
+                        }
                     }
                 ) {
                     ${releaseFields}
@@ -110,6 +114,7 @@ function initNetworksActions(store, _router) {
             before: store.getters.before,
             afterTime: store.getters.after,
             beforeTime: store.getters.before,
+            isNew: store.getters.isNew,
             orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
             exclude: store.state.ui.filter,
         });

assets/js/releases/actions.js

@@ -9,6 +9,7 @@ function initReleasesActions(store, _router) {
                 $limit:Int = 1000,
                 $after:Date = "1900-01-01",
                 $before:Date = "2100-01-01",
+                $isNew:[Boolean!] = [true,false]
                 $orderBy:[ReleasesOrderBy!],
                 $exclude: [String!]
             ) {
@@ -18,6 +19,7 @@ function initReleasesActions(store, _router) {
             limit,
             after: store.getters.after,
             before: store.getters.before,
+            isNew: store.getters.isNew,
             orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
             exclude: store.state.ui.filter,
         });
@@ -41,7 +43,7 @@ function initReleasesActions(store, _router) {
                         slug
                         date
                         url
-                        batch
+                        isNew
                         site {
                             id
                             slug

assets/js/sites/actions.js

@@ -10,6 +10,7 @@ function initSitesActions(store, _router) {
                 $limit:Int = 100,
                 $after:Date = "1900-01-01",
                 $before:Date = "2100-01-01",
+                $isNew:[Boolean!] = [true,false]
                 $orderBy:[ReleasesOrderBy!]
                 $exclude: [String!]
             ) {
@@ -38,6 +39,7 @@ function initSitesActions(store, _router) {
             limit,
             after: store.getters.after,
             before: store.getters.before,
+            isNew: store.getters.isNew,
             orderBy: store.state.ui.range === 'upcoming' ? 'DATE_ASC' : 'DATE_DESC',
             exclude: store.state.ui.filter,
         });

assets/js/tags/actions.js

@@ -12,6 +12,7 @@ function initTagsActions(store, _router) {
                 $limit:Int = 1000,
                 $after:Date = "1900-01-01",
                 $before:Date = "2100-01-01",
+                $isNew:[Boolean!] = [true,false]
                 $orderBy: [ReleasesTagsOrderBy!],
                 $exclude: [String!]
             ) {
@@ -56,6 +57,9 @@ function initTagsActions(store, _router) {
                                         }
                                     }
                                 }
+                                isNew: {
+                                    in: $isNew
+                                }
                             }
                         },
                         first: $limit,
@@ -72,6 +76,7 @@ function initTagsActions(store, _router) {
             limit,
             after: store.getters.after,
             before: store.getters.before,
+            isNew: store.getters.isNew,
             orderBy: store.state.ui.range === 'upcoming' ? 'RELEASE_BY_RELEASE_ID__DATE_ASC' : 'RELEASE_BY_RELEASE_ID__DATE_DESC',
             exclude: store.state.ui.filter,
         });

assets/js/ui/actions.js

@@ -11,9 +11,15 @@ function initUiActions(_store, _router) {
         localStorage.setItem('range', range);
     }
 
+    function setBatch({ commit }, batch) {
+        commit('setBatch', batch);
+        localStorage.setItem('batch', batch);
+    }
+
     return {
         setFilter,
         setRange,
+        setBatch,
     };
 }
 

assets/js/ui/getters.js

@@ -1,7 +1,7 @@
 import dayjs from 'dayjs';
 
 const dateRanges = {
-    new: () => ({
+    latest: () => ({
         after: '1900-01-01',
         before: dayjs(new Date()).add(1, 'day').format('YYYY-MM-DD'),
     }),
@@ -27,8 +27,13 @@ function after(state) {
     return dateRanges[state.range]().after;
 }
 
+function isNew(state) {
+    return state.batch === 'new' ? [true] : [true, false];
+}
+
 export default {
     rangeDates,
     before,
     after,
+    isNew,
 };

assets/js/ui/mutations.js

@@ -6,7 +6,12 @@ function setRange(state, range) {
     state.range = range;
 }
 
+function setBatch(state, batch) {
+    state.batch = batch;
+}
+
 export default {
     setFilter,
     setRange,
+    setBatch,
 };

assets/js/ui/state.js

@@ -1,7 +1,9 @@
 const storedFilter = localStorage.getItem('filter');
 const storedRange = localStorage.getItem('range');
+const storedBatch = localStorage.getItem('batch');
 
 export default {
     filter: storedFilter ? storedFilter.split(',') : ['gay', 'transsexual'],
-    range: storedRange || 'new',
+    range: storedRange || 'latest',
+    batch: storedBatch || 'all',
 };

migrations/20190325001339_releases.js

@@ -327,6 +327,13 @@ exports.up = knex => Promise.resolve()
         table.datetime('created_at')
             .defaultTo(knex.fn.now());
     }))
+    .then(() => knex.schema.createTable('batches', (table) => {
+        table.increments('id', 12);
+        table.text('comment');
+
+        table.datetime('created_at')
+            .defaultTo(knex.fn.now());
+    }))
     .then(() => knex.schema.createTable('releases', (table) => {
         table.increments('id', 16);
 
@@ -362,7 +369,14 @@ exports.up = knex => Promise.resolve()
         table.boolean('deep');
         table.string('deep_url', 1000);
 
-        table.string('batch');
+        table.integer('created_batch_id', 12)
+            .references('id')
+            .inTable('batches');
+
+        table.integer('updated_batch_id', 12)
+            .references('id')
+            .inTable('batches');
+
         table.datetime('created_at')
             .defaultTo(knex.fn.now());
     }))
@@ -508,6 +522,10 @@ exports.up = knex => Promise.resolve()
             url ILIKE ('%' || search || '%')
         $$ LANGUAGE SQL STABLE;
 
+        CREATE FUNCTION releases_is_new(release releases) RETURNS boolean AS $$
+            SELECT NOT EXISTS(SELECT true FROM batches WHERE batches.id = release.created_batch_id + 1 LIMIT 1);
+        $$ LANGUAGE sql STABLE;
+
         COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
         COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
     `));
@@ -527,6 +545,7 @@ exports.down = knex => knex.raw(`
     DROP TABLE IF EXISTS releases_teasers CASCADE;
     DROP TABLE IF EXISTS releases_tags CASCADE;
     DROP TABLE IF EXISTS releases_search CASCADE;
+    DROP TABLE IF EXISTS batches CASCADE;
     DROP TABLE IF EXISTS actors_avatars CASCADE;
     DROP TABLE IF EXISTS actors_photos CASCADE;
     DROP TABLE IF EXISTS actors_social CASCADE;

src/media.js

@@ -271,7 +271,7 @@ function groupItems(items) {
 }
 
 async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
-    const presentSources = sources.filter(Boolean);
+    const presentSources = sources.filter(source => typeof source === 'string' || Array.isArray(source) || (source && source.src));
 
     if (presentSources.length === 0) {
         return {};

src/releases.js

@@ -202,7 +202,7 @@ async function attachStudio(release) {
     };
 }
 
-async function curateReleaseEntry(release) {
+async function curateReleaseEntry(release, batchId, existingRelease) {
     const slug = slugify(release.title, {
         encode: true,
         limit: config.titleSlugLength,
@@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
         // rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
         deep: typeof release.deep === 'boolean' ? release.deep : false,
         deep_url: release.deepUrl,
+        updated_batch_id: batchId,
+        ...(!existingRelease && { created_batch_id: batchId }),
     };
 
     return curatedRelease;
@@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
                 sites.slug || ' ' ||
                 networks.name || ' ' ||
                 networks.slug || ' ' ||
+                releases.shoot_id || ' ' ||
                 EXTRACT(YEAR FROM releases.date) || ' ' ||
                 CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
                 CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
     }
 }
 
-async function storeRelease(release) {
+async function storeRelease(release, batchId) {
     const existingRelease = await knex('releases')
         .where({
             entry_id: release.entryId,
@@ -416,7 +419,7 @@ async function storeRelease(release) {
         })
         .first();
 
-    const curatedRelease = await curateReleaseEntry(release);
+    const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
 
     if (existingRelease && !argv.redownload) {
         return existingRelease;
@@ -453,11 +456,13 @@ async function storeRelease(release) {
 }
 
 async function storeReleases(releases) {
+    const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+
     const storedReleases = await Promise.map(releases, async (release) => {
         try {
             const releaseWithChannelSite = await attachChannelSite(release);
             const releaseWithStudio = await attachStudio(releaseWithChannelSite);
-            const { id, slug } = await storeRelease(releaseWithStudio);
+            const { id, slug } = await storeRelease(releaseWithStudio, batchId);
 
             return {
                 id,

src/scrapers/private.js

@@ -84,12 +84,12 @@ async function scrapeScene(html, url, site) {
     release.likes = Number($('.content-desc #social-actions #likes').text());
 
     const posterScript = $('script:contains(poster)').html();
-    const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
+    const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
-    release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
+    release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src');
 
-    release.trailer = {
+    const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src');
-        src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
+
-    };
+    if (trailer) release.trailer = { src: trailer };
 
     release.photos = await getPhotos(release.entryId, site);
     release.movie = $('a[data-track="FULL MOVIE"]').attr('href');

src/web/plugins/releases.js

@@ -4,8 +4,14 @@ const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
 
 const schemaExtender = makeExtendSchemaPlugin(_build => ({
     typeDefs: gql`
+      extend type Release {}
     `,
     resolvers: {
+        Release: {
+            async foo(_parent, _args, _context, _info) {
+                // template
+            },
+        },
     },
 }));