Updated GraphQL queries to Datetime. Updated template to latest guideline.

This commit is contained in:
DebaucheryLibrarian 2020-07-16 15:55:03 +02:00
parent d0e61978d6
commit 66d6322c1d
10 changed files with 89 additions and 39 deletions

View File

@ -1 +0,0 @@
src/scrapers/template.js

View File

@ -212,6 +212,7 @@ export default {
.sidebar-toggle { .sidebar-toggle {
display: none; display: none;
align-items: center;
height: 100%; height: 100%;
.logo { .logo {

View File

@ -183,7 +183,7 @@ export default {
height: 100%; height: 100%;
color: var(--text); color: var(--text);
background: var(--background); background: var(--background);
box-shadow: 0 0 3px var(--darken); box-shadow: 0 0 3px var(--darken-weak);
} }
.sidebar-header { .sidebar-header {

View File

@ -24,8 +24,8 @@ function initActorActions(store, router) {
$actorId: Int! $actorId: Int!
$limit:Int = 10, $limit:Int = 10,
$offset:Int = 0, $offset:Int = 0,
$after:Date = "1900-01-01", $after:Datetime = "1900-01-01",
$before:Date = "2100-01-01", $before:Datetime = "2100-01-01",
$orderBy:[ReleasesActorsOrderBy!] $orderBy:[ReleasesActorsOrderBy!]
$selectableTags: [String], $selectableTags: [String],
$excludeTags: [String!] $excludeTags: [String!]

View File

@ -20,8 +20,8 @@ function initEntitiesActions(store, _router) {
$entityType: String! = "channel" $entityType: String! = "channel"
$limit: Int = 10, $limit: Int = 10,
$offset: Int = 0, $offset: Int = 0,
$after: Date = "1900-01-01", $after: Datetime = "1900-01-01",
$before: Date = "2100-01-01", $before: Datetime = "2100-01-01",
$afterTime: Datetime = "1900-01-01", $afterTime: Datetime = "1900-01-01",
$beforeTime: Datetime = "2100-01-01", $beforeTime: Datetime = "2100-01-01",
$orderBy: [ReleasesOrderBy!] $orderBy: [ReleasesOrderBy!]

View File

@ -11,8 +11,8 @@ function initReleasesActions(store, _router) {
query Releases( query Releases(
$limit:Int = 1000, $limit:Int = 1000,
$offset:Int = 0, $offset:Int = 0,
$after:Date = "1900-01-01", $after:Datetime = "1900-01-01",
$before:Date = "2100-01-01", $before:Datetime = "2100-01-01",
$orderBy:[ReleasesOrderBy!], $orderBy:[ReleasesOrderBy!],
$exclude: [String!] $exclude: [String!]
) { ) {

View File

@ -13,8 +13,8 @@ function initTagsActions(store, _router) {
query Tag( query Tag(
$tagSlug:String! $tagSlug:String!
$limit:Int = 1000, $limit:Int = 1000,
$after:Date = "1900-01-01", $after:Datetime = "1900-01-01",
$before:Date = "2100-01-01", $before:Datetime = "2100-01-01",
$orderBy: [ReleasesTagsOrderBy!], $orderBy: [ReleasesTagsOrderBy!],
$exclude: [String!] $exclude: [String!]
) { ) {

View File

@ -505,6 +505,8 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
try { try {
const scraper = scrapers[scraperSlug]; const scraper = scrapers[scraperSlug];
const context = { const context = {
...entitiesBySlug[scraperSlug],
// legacy
site: entitiesBySlug[scraperSlug] || null, site: entitiesBySlug[scraperSlug] || null,
network: entitiesBySlug[scraperSlug] || null, network: entitiesBySlug[scraperSlug] || null,
entity: entitiesBySlug[scraperSlug] || null, entity: entitiesBySlug[scraperSlug] || null,

View File

@ -15,6 +15,7 @@ const cherrypimps = require('./cherrypimps');
const ddfnetwork = require('./ddfnetwork'); const ddfnetwork = require('./ddfnetwork');
const digitalplayground = require('./digitalplayground'); const digitalplayground = require('./digitalplayground');
const dogfart = require('./dogfart'); const dogfart = require('./dogfart');
const elegantangel = require('./elegantangel');
const evilangel = require('./evilangel'); const evilangel = require('./evilangel');
const fakehub = require('./fakehub'); const fakehub = require('./fakehub');
const famedigital = require('./famedigital'); const famedigital = require('./famedigital');
@ -90,6 +91,7 @@ module.exports = {
digitalplayground, digitalplayground,
dogfart, dogfart,
dogfartnetwork: dogfart, dogfartnetwork: dogfart,
elegantangel,
evilangel, evilangel,
fakehub, fakehub,
famedigital, famedigital,

View File

@ -1,51 +1,97 @@
'use strict'; 'use strict';
const { get, geta } = require('../utils/q'); const qu = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeLatest(scenes, site) { function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => { return scenes.map(({ query }) => {
const release = {}; const release = {};
const pathname = query.url('.title a');
release.title = qu.q('.title a', true); release.entryId = pathname.match(/\/scene\/(\d+)/)[1];
release.url = `${site.url}${pathname}`;
const pathname = qu.url('.title a'); release.title = query.cnt('.title a');
release.entryId = pathname.split('/')[3]; release.description = query.cnt('.description');
release.url = `${site.url}${pathname}`;
release.date = qu.date('.date', 'MMM DD, YYYY'); release.date = query.date('.date', 'MMM DD, YYYY');
release.actors = qu.all('.models a.model', true); release.actors = query.cnts('.models a.model');
release.poster = qu.q('img').dataset.original; release.poster = query.q('img.poster');
release.stars = Number(qu.q('.rating', true)); release.stars = query.number('.rating');
release.likes = Number(qu.q('.likes', true)); release.likes = query.number('.likes');
console.log(release); console.log(release);
return release; return release;
}); });
} }
function scrapeScene({ qu }, _site) { function scrapeScene({ query }) {
const release = {}; const release = {};
console.log(release); release.title = query.cnt('h3.title');
return release; release.description = query.cnt('p.description');
[release.poster, ...release.photos] = query.imgs('.preview-thumb');
const trailer = query.video('.trailer video');
release.trailer = { src: trailer };
console.log(release);
return release;
} }
async function fetchLatest(site, page = 1) { function scrapeProfile({ query, el }, actorName, entity, include) {
const url = `${site.url}/${page}`; const profile = {};
const res = await geta(url, '.selector');
return res.ok ? scrapeLatest(res.items, site) : res.status; profile.description = query.cnt('.bio-text');
profile.birthPlace = query.cnt('.birth-place span');
profile.avatar = query.img('.actor-photo img');
if (include.releases) {
return scrapeAll(qu.initAll(el, '.scene'));
}
console.log(profile);
return profile;
} }
async function fetchScene(url, site) { async function fetchLatest(channel, page = 1) {
const res = await get(url); const url = `${channel.url}/${page}`;
const res = await qu.getAll(url, '.scene');
return res.ok ? scrapeScene(res.item, site) : res.status; if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, channel);
}
return res.status;
}
async function fetchProfile(actorName, entity, include) {
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
const res = await qu.get(url);
if (res.ok) {
return scrapeProfile(res.item, actorName, entity, include);
}
return res.status;
} }
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile,
}; };