diff --git a/assets/components/actors/actor.vue b/assets/components/actors/actor.vue index 1d4ac2ce..adcc697d 100644 --- a/assets/components/actors/actor.vue +++ b/assets/components/actors/actor.vue @@ -29,7 +29,7 @@ /> -
+
diff --git a/assets/components/header/header.vue b/assets/components/header/header.vue index 08975241..d87ce523 100644 --- a/assets/components/header/header.vue +++ b/assets/components/header/header.vue @@ -356,7 +356,7 @@ export default { } } -@media(max-width: $breakpoint-micro) { +@media(max-width: $breakpoint) { .nav { display: none; } diff --git a/assets/components/movies/movies.vue b/assets/components/movies/movies.vue index 78c7e3b1..7d20ac09 100644 --- a/assets/components/movies/movies.vue +++ b/assets/components/movies/movies.vue @@ -1,11 +1,48 @@ + + diff --git a/assets/components/movies/tile.vue b/assets/components/movies/tile.vue new file mode 100644 index 00000000..7a6a72d3 --- /dev/null +++ b/assets/components/movies/tile.vue @@ -0,0 +1,38 @@ + + + + + diff --git a/assets/components/tags/tags.vue b/assets/components/tags/tags.vue index e2f9de62..ddf64c70 100644 --- a/assets/components/tags/tags.vue +++ b/assets/components/tags/tags.vue @@ -48,13 +48,11 @@ async function mounted() { 'creampie', 'squirting', ], - ethnicity: [ + appearance: [ 'asian', 'ebony', 'latina', 'caucasian', - ], - appearance: [ 'natural-boobs', 'fake-boobs', 'blonde', diff --git a/assets/components/tags/tile.vue b/assets/components/tags/tile.vue index 07237dc3..eb784567 100644 --- a/assets/components/tags/tile.vue +++ b/assets/components/tags/tile.vue @@ -3,12 +3,6 @@ v-if="tag.poster" class="tile" > - {{ tag.name }} - + + {{ tag.name }} +
traxxx - - @@ -18,6 +16,8 @@ + + diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js index b002c5e9..905c4911 100644 --- a/assets/js/entities/actions.js +++ b/assets/js/entities/actions.js @@ -180,6 +180,13 @@ function initEntitiesActions(store, _router) { type: { equalTo: "network" } + childEntitiesConnection: { + some: { + type: { + equalTo: "channel" + } + } + } } { independent: { @@ -197,7 +204,7 @@ function initEntitiesActions(store, _router) { in: $entitySlugs } } - ] + ], } ) { id diff --git a/assets/js/fragments.js b/assets/js/fragments.js index e3919d1f..024d41df 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -243,11 +243,6 @@ const releaseFragment = ` ${releaseTrailerFragment} ${releaseTeaserFragment} ${siteFragment} - scenes: releasesMoviesByMovieId { - scene { - ${releaseFields} - } - } studio { id name diff --git a/assets/js/releases/actions.js b/assets/js/releases/actions.js index 6f4a24ef..fae64299 100644 --- a/assets/js/releases/actions.js +++ b/assets/js/releases/actions.js @@ -47,9 +47,48 @@ function initReleasesActions(store, _router) { return curateRelease(release); } + async function fetchMovies({ _commit }, { limit = 10, pageNumber = 1 }) { + const { connection: { movies, totalCount } } = await graphql(` + query Movies( + $limit:Int = 1000, + $offset:Int = 0, + ) { + connection: moviesConnection( + first: $limit + offset: $offset + orderBy: DATE_ASC + ) { + movies: nodes { + id + title + url + slug + date + datePrecision + entity { + id + name + slug + } + } + totalCount + } + } + `, { + limit, + offset: Math.max(0, (pageNumber - 1)) * limit, + }); + + return { + movies: movies.map(release => curateRelease(release)), + totalCount, + }; + } + return { fetchReleases, fetchReleaseById, + fetchMovies, }; } diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 3b6a29d0..02000922 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -643,6 +643,32 @@ exports.up = knex => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); })) + .then(() => knex.schema.createTable('movies_covers', (table) => { + table.integer('movie_id', 16) + .notNullable() + .references('id') + .inTable('movies'); + + table.text('media_id', 21) + .notNullable() + .references('id') + .inTable('media'); + + table.unique(['movie_id', 'media_id']); + })) + .then(() => knex.schema.createTable('movies_trailers', (table) => { + table.integer('movie_id', 16) + .notNullable() + .references('id') + .inTable('movies'); + + table.text('media_id', 21) + .notNullable() + .references('id') + .inTable('media'); + + table.unique('movie_id'); + })) .then(() => knex.schema.createTable('releases', (table) => { table.increments('id', 16); @@ -965,6 +991,9 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS releases_tags CASCADE; DROP TABLE IF EXISTS releases_search CASCADE; + DROP TABLE IF EXISTS movies_covers CASCADE; + DROP TABLE IF EXISTS movies_trailers CASCADE; + DROP TABLE IF EXISTS batches CASCADE; DROP TABLE IF EXISTS actors_avatars CASCADE; diff --git a/public/img/tags/bukkake/0.jpeg b/public/img/tags/bukkake/0.jpeg index f1da17ca..f0426056 100644 Binary files a/public/img/tags/bukkake/0.jpeg and b/public/img/tags/bukkake/0.jpeg differ diff --git a/public/img/tags/bukkake/lazy/0.jpeg b/public/img/tags/bukkake/lazy/0.jpeg index a3c3c0eb..ad5be3e8 100644 Binary files a/public/img/tags/bukkake/lazy/0.jpeg and b/public/img/tags/bukkake/lazy/0.jpeg differ diff --git a/public/img/tags/bukkake/thumbs/0.jpeg b/public/img/tags/bukkake/thumbs/0.jpeg index c199e740..d206770b 100644 Binary files a/public/img/tags/bukkake/thumbs/0.jpeg and b/public/img/tags/bukkake/thumbs/0.jpeg differ diff --git a/public/img/tags/bukkake/thumbs/poster.jpeg b/public/img/tags/bukkake/thumbs/poster.jpeg index b6db591b..5366c8ea 100644 Binary files a/public/img/tags/bukkake/thumbs/poster.jpeg and b/public/img/tags/bukkake/thumbs/poster.jpeg differ diff --git a/public/img/tags/facefucking/1.jpeg b/public/img/tags/facefucking/1.jpeg index 468037eb..9966187d 100644 Binary files a/public/img/tags/facefucking/1.jpeg and b/public/img/tags/facefucking/1.jpeg differ diff --git a/public/img/tags/facefucking/lazy/1.jpeg b/public/img/tags/facefucking/lazy/1.jpeg index 53cb7ad1..6d02c2c2 100644 Binary files a/public/img/tags/facefucking/lazy/1.jpeg and b/public/img/tags/facefucking/lazy/1.jpeg differ diff --git a/public/img/tags/facefucking/thumbs/1.jpeg b/public/img/tags/facefucking/thumbs/1.jpeg index f7deb5e1..3df047aa 100644 Binary files a/public/img/tags/facefucking/thumbs/1.jpeg and b/public/img/tags/facefucking/thumbs/1.jpeg differ diff --git a/public/img/tags/facefucking/thumbs/2.jpeg b/public/img/tags/facefucking/thumbs/2.jpeg index 0d08a5e6..a61f1bb7 100644 Binary files a/public/img/tags/facefucking/thumbs/2.jpeg and b/public/img/tags/facefucking/thumbs/2.jpeg differ diff --git a/public/img/tags/facefucking/thumbs/3.jpeg b/public/img/tags/facefucking/thumbs/3.jpeg index d1029d2b..88cbc1be 100644 Binary files a/public/img/tags/facefucking/thumbs/3.jpeg and b/public/img/tags/facefucking/thumbs/3.jpeg differ diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 4e672638..deeece25 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -7920,7 +7920,7 @@ const sites = [ { name: 'PornDoe Pedia', slug: 'porndoepedia', - url: 'https://vipsexvault.com/channels/vipsexvault-pedia.en.html', + url: 'https://vipsexvault.com/channels/porndoe-pedia.en.html', parent: 'vipsexvault', }, { diff --git a/seeds/04_media.js b/seeds/04_media.js index 6c054e17..947baf95 100644 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -615,7 +615,7 @@ const tagPosters = [ ['dvp', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'], ['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'], ['ebony', 2, 'Nia Nacci for Sweetheart Video'], - ['facefucking', 2, 'Jynx Maze for Throated'], + ['facefucking', 1, 'Paige Owens in "Dark Meat 12" for Evil Angel'], ['facial', 0, 'Brooklyn Gray in "All About Ass 4" for Evil Angel'], ['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'], ['family', 0, 'Teanna Trump in "A Family Appear: Part One" for Brazzers'], @@ -703,8 +703,8 @@ const tagPhotos = [ ['ebony', 1, 'Ana Foxxx in "DP Me 4" for HardX'], ['facial', 2, 'Ashly Anderson for Hookup Hotshot'], ['facial', 'poster', 'Jynx Maze'], + ['facefucking', 2, 'Jynx Maze for Throated'], ['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'], - ['facefucking', 1, 'Carrie for Young Throats'], ['fake-boobs', 7, 'Madison Ivy for Passion HD'], ['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'], ['fake-boobs', 6, 'Cathy Heaven in "Heavenly Ass" for Big Wett Butts'], diff --git a/src/app.js b/src/app.js index 52aeff44..0f7fdbd3 100644 --- a/src/app.js +++ b/src/app.js @@ -8,7 +8,7 @@ const initServer = require('./web/server'); const knex = require('./knex'); const fetchUpdates = require('./updates'); const { fetchScenes, fetchMovies } = require('./deep'); -const { storeReleases, updateReleasesSearch } = require('./store-releases'); +const { storeReleases, storeMovies, updateReleasesSearch } = require('./store-releases'); const { scrapeActors } = require('./actors'); const getFileEntries = require('./utils/file-entries'); @@ -31,14 +31,14 @@ async function init() { const updateBaseScenes = (argv.all || argv.channels || argv.networks || argv.movies) && await fetchUpdates(); const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile); - const sceneUrls = (argv.scenes || []).concat(scenesFromFile || []); + const sceneUrls = (argv.scene || []).concat(scenesFromFile || []); const deepScenes = argv.deep ? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])]) : [...(updateBaseScenes || []), ...(actorBaseScenes || [])]; - const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean); - const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]); + const sceneMovies = deepScenes && argv.movie && deepScenes.map(scene => scene.movie).filter(Boolean); + const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]); if (argv.inspect) { console.log(util.inspect(deepScenes)); @@ -46,10 +46,15 @@ async function init() { } if (argv.save) { - await storeReleases([ - ...(deepScenes || []), - ...(deepMovies || []), - ]); + if (deepScenes.length > 0) { + await storeReleases(deepScenes); + } + + console.log(deepMovies); + + if (deepMovies.length > 0) { + await storeMovies(deepMovies); + } } knex.destroy(); diff --git a/src/argv.js b/src/argv.js index f6ec567d..d01eb009 100644 --- a/src/argv.js +++ b/src/argv.js @@ -25,10 +25,6 @@ const { argv } = yargs type: 'array', alias: 'channel', }) - .option('movies', { - describe: 'Scrape movies from channels', - type: 'array', - }) .option('actors', { describe: 'Scrape actors by name or slug', type: 'array', @@ -61,19 +57,18 @@ const { argv } = yargs alias: 'with-profiles', default: false, }) - .option('scenes', { + .option('scene', { describe: 'Scrape scene info from URL', type: 'array', - alias: 'scene', }) - .option('scenes-file', { + .option('scene-file', { describe: 'Scrape scene info from URLs in a file', type: 'string', + alias: 'scenes-file', }) .option('movie', { describe: 'Scrape movie info from URL', type: 'array', - alias: 'movies', }) .option('sources', { describe: 'Use these scrapers for actor data', @@ -88,12 +83,17 @@ const { argv } = yargs .option('latest', { describe: 'Scrape latest releases if available', type: 'boolean', - default: true, + default: false, }) .option('upcoming', { describe: 'Scrape upcoming releases if available', type: 'boolean', - default: true, + default: false, + }) + .option('movies', { + describe: 'Scrape movies from channels', + type: 'boolean', + default: false, }) .option('force', { describe: 'Don\'t ignore duplicates, update existing entries', diff --git a/src/media.js b/src/media.js index 766e75de..7d80c352 100644 --- a/src/media.js +++ b/src/media.js @@ -612,7 +612,7 @@ async function storeMedias(baseMedias) { return [...newMediaWithEntries, ...existingHashMedias]; } -async function associateReleaseMedia(releases) { +async function associateReleaseMedia(releases, type = 'releases') { if (!argv.media) { return; } @@ -670,7 +670,7 @@ async function associateReleaseMedia(releases) { .filter(Boolean); if (associations.length > 0) { - await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`); + await knex.raw(`${knex(`${type}_${role}`).insert(associations)} ON CONFLICT DO NOTHING`); } }, Promise.resolve()); } diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index 366ce6b7..ba1e31bf 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -276,13 +276,13 @@ async function scrapeScene({ html, qu }, url, site, include) { return release; } -function scrapeMovie({ el, qu }, url, site) { +function scrapeMovie({ el, query }, url, site) { const movie = { url, site }; - movie.entryId = qu.q('.dvd_details_overview .rating_box').dataset.id; - movie.title = qu.q('.title_bar span', true); - movie.covers = qu.urls('#dvd-cover-flip > a'); - movie.channel = slugify(qu.q('.update_date a', true), ''); + movie.entryId = query.q('.rating_box').dataset.id; + movie.title = query.q('.title_bar span', true); + movie.covers = query.urls('#dvd-cover-flip > a'); + movie.channel = slugify(query.q('.update_date a', true), ''); // movie.releases = Array.from(document.querySelectorAll('.cell.dvd_info > a'), el => el.href); const sceneQus = ctxa(el, '.dvd_details'); diff --git a/src/store-releases.js b/src/store-releases.js index 7d30a411..35c9195c 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -12,7 +12,7 @@ const { associateReleaseTags } = require('./tags'); const { curateEntity } = require('./entities'); const { associateReleaseMedia } = require('./media'); -function curateReleaseEntry(release, batchId, existingRelease) { +function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { const slugBase = release.title || (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`) || (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`) @@ -23,19 +23,18 @@ function curateReleaseEntry(release, batchId, existingRelease) { limit: config.titleSlugLength, }); + console.log(release); + const curatedRelease = { title: release.title, entry_id: release.entryId || null, entity_id: release.entity.id, studio_id: release.studio?.id || null, - shoot_id: release.shootId || null, url: release.url, date: Number(release.date) ? release.date : null, - production_date: Number(release.productionDate) ? release.productionDate : null, date_precision: release.datePrecision, slug, description: release.description, - duration: release.duration, comment: release.comment, // director: release.director, // likes: release.rating && release.rating.likes, @@ -46,6 +45,12 @@ function curateReleaseEntry(release, batchId, existingRelease) { updated_batch_id: batchId, }; + if (type === 'scene') { + curatedRelease.shoot_id = release.shootId || null; + curatedRelease.productionDate = Number(release.productionDate) ? release.productionDate : null; + curatedRelease.duration = release.duration; + } + if (!existingRelease && !release.id) { curatedRelease.created_batch_id = batchId; } @@ -252,7 +257,24 @@ async function storeReleases(releases) { return releasesWithId; } +async function storeMovies(movies) { + const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); + + console.log(movies); + + const curatedMovieEntries = movies.map(release => curateReleaseEntry(release, batchId, null, 'movie')); + console.log(curatedMovieEntries); + const storedMovies = await knex.batchInsert('movies', curatedMovieEntries).returning('*'); + + const moviesWithId = attachReleaseIds(movies, storedMovies); + + await associateReleaseMedia(moviesWithId, 'movies'); + + return storedMovies; +} + module.exports = { storeReleases, + storeMovies, updateReleasesSearch, }; diff --git a/src/updates.js b/src/updates.js index 0d9009b9..9ec3d642 100644 --- a/src/updates.js +++ b/src/updates.js @@ -161,6 +161,21 @@ async function scrapeUpcomingReleases(scraper, entity, preData) { return []; } +async function scrapeMovies(scraper, entity) { + if (!scraper.fetchMovies) { + return []; + } + + try { + // return await scrapeReleases(scraper, entity, preData, true); + return await scraper.fetchMovies(entity); + } catch (error) { + logger.warn(`Failed to scrape movies for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); + } + + return []; +} + async function scrapeChannelReleases(scraper, channelEntity, preData) { const [latestReleases, upcomingReleases] = await Promise.all([ argv.latest @@ -169,12 +184,11 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) { argv.upcoming ? scrapeUpcomingReleases(scraper, channelEntity, preData) : [], + argv.movies + ? scrapeMovies(scraper, channelEntity, preData) + : [], ]); - if (scraper.fetchMovies) { - await scraper.fetchMovies(channelEntity); - } - logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`); return [...latestReleases, ...upcomingReleases];