Storing actor avatars. Using 1 second interval queue for location resolve as per OSM code of conduct.
This commit is contained in:
parent
21d4dd6bfa
commit
05ee57378a
|
|
@ -36,18 +36,15 @@ function curateActor(actor) {
|
|||
updatedAt: new Date(actor.updatedAt),
|
||||
};
|
||||
|
||||
if (actor.avatar) {
|
||||
curatedActor.avatar = actor.avatar.media;
|
||||
if (actor.profiles && actor.profiles.length > 0) {
|
||||
curatedActor.avatar = actor.profiles.slice(0, 1)[0].avatar;
|
||||
curatedActor.photos = actor.profiles.slice(1).map(profile => profile.avatar);
|
||||
}
|
||||
|
||||
if (actor.releases) {
|
||||
curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
|
||||
}
|
||||
|
||||
if (actor.photos) {
|
||||
curatedActor.photos = actor.photos.map(photo => photo.media);
|
||||
}
|
||||
|
||||
return curatedActor;
|
||||
}
|
||||
|
||||
|
|
@ -99,21 +96,14 @@ function initActorActions(store, _router) {
|
|||
name
|
||||
slug
|
||||
}
|
||||
actorsProfiles {
|
||||
actorsAvatarByProfileId {
|
||||
media {
|
||||
path
|
||||
thumbnail
|
||||
copyright
|
||||
}
|
||||
}
|
||||
}
|
||||
photos: actorsPhotos {
|
||||
media {
|
||||
profiles: actorsProfiles {
|
||||
description
|
||||
avatar: avatarMedia {
|
||||
id
|
||||
thumbnail
|
||||
path
|
||||
index
|
||||
thumbnail
|
||||
lazy
|
||||
comment
|
||||
copyright
|
||||
}
|
||||
}
|
||||
|
|
@ -239,6 +229,7 @@ function initActorActions(store, _router) {
|
|||
actorsProfiles {
|
||||
actorsAvatarByProfileId {
|
||||
media {
|
||||
id
|
||||
path
|
||||
thumbnail
|
||||
copyright
|
||||
|
|
|
|||
|
|
@ -59,13 +59,16 @@ exports.up = knex => Promise.resolve()
|
|||
.inTable('media')
|
||||
.unique();
|
||||
}))
|
||||
.then(() => knex.raw(`
|
||||
.then(() => { // eslint-disable-line arrow-body-style
|
||||
// allow vim fold
|
||||
return knex.raw(`
|
||||
CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
|
||||
SELECT media_id FROM media_sfw
|
||||
ORDER BY random()
|
||||
LIMIT 1;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
`))
|
||||
`);
|
||||
})
|
||||
.then(() => knex.schema.alterTable('media', (table) => {
|
||||
table.string('sfw_media_id', 21)
|
||||
.references('id')
|
||||
|
|
@ -140,6 +143,7 @@ exports.up = knex => Promise.resolve()
|
|||
table.increments('id', 12);
|
||||
|
||||
table.string('name');
|
||||
table.string('alias');
|
||||
table.string('url');
|
||||
table.text('description');
|
||||
table.json('parameters');
|
||||
|
|
@ -322,6 +326,7 @@ exports.up = knex => Promise.resolve()
|
|||
table.increments('id', 12);
|
||||
|
||||
table.integer('actor_id', 12)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('actors');
|
||||
|
||||
|
|
@ -371,6 +376,10 @@ exports.up = knex => Promise.resolve()
|
|||
table.string('piercings');
|
||||
table.string('tattoos');
|
||||
|
||||
table.string('avatar_media_id', 21)
|
||||
.references('id')
|
||||
.inTable('media');
|
||||
|
||||
table.datetime('scraped_at');
|
||||
table.boolean('scrape_success');
|
||||
|
||||
|
|
@ -386,7 +395,9 @@ exports.up = knex => Promise.resolve()
|
|||
|
||||
table.string('name');
|
||||
}))
|
||||
.then(() => knex('body').insert([
|
||||
.then(() => { // eslint-disable-line arrow-body-style
|
||||
// allow vim fold
|
||||
return knex('body').insert([
|
||||
// head
|
||||
{ slug: 'head', name: 'head' },
|
||||
{ slug: 'face', name: 'face' },
|
||||
|
|
@ -501,7 +512,8 @@ exports.up = knex => Promise.resolve()
|
|||
{ slug: 'middle-toe', name: 'middle toe' },
|
||||
{ slug: 'fourth-toe', name: 'fourth toe' },
|
||||
{ slug: 'little-toe', name: 'little toe' },
|
||||
]))
|
||||
]);
|
||||
})
|
||||
.then(() => knex.schema.createTable('actors_tattoos', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
|
|
@ -769,7 +781,9 @@ exports.up = knex => Promise.resolve()
|
|||
.references('id')
|
||||
.inTable('releases');
|
||||
}))
|
||||
.then(() => knex.raw(`
|
||||
.then(() => { // eslint-disable-line arrow-body-style
|
||||
// allow vim fold
|
||||
return knex.raw(`
|
||||
ALTER TABLE releases
|
||||
ADD CONSTRAINT ensure_site_or_network CHECK (site_id IS NOT NULL OR network_id IS NOT NULL);
|
||||
|
||||
|
|
@ -828,9 +842,12 @@ exports.up = knex => Promise.resolve()
|
|||
|
||||
COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
|
||||
COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
|
||||
`));
|
||||
`);
|
||||
});
|
||||
|
||||
exports.down = knex => knex.raw(`
|
||||
exports.down = (knex) => { // eslint-disable-line arrow-body-style
|
||||
// allow vim fold
|
||||
return knex.raw(`
|
||||
DROP VIEW IF EXISTS movie_actors;
|
||||
DROP VIEW IF EXISTS movie_tags;
|
||||
|
||||
|
|
@ -880,3 +897,4 @@ exports.down = knex => knex.raw(`
|
|||
DROP TEXT SEARCH CONFIGURATION IF EXISTS traxxx;
|
||||
DROP TEXT SEARCH DICTIONARY IF EXISTS traxxx_dict;
|
||||
`);
|
||||
};
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 140 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 151 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
|
|
@ -0,0 +1,84 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 547 112" enable-background="new 0 0 547 112" xml:space="preserve">
|
||||
<radialGradient id="SVGID_1_" cx="171.1188" cy="60.5308" r="34.3229" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.1989" style="stop-color:#EE2724"/>
|
||||
<stop offset="0.3065" style="stop-color:#EE2424"/>
|
||||
<stop offset="1" style="stop-color:#CD2027"/>
|
||||
</radialGradient>
|
||||
<polygon fill="url(#SVGID_1_)" points="182.1,18.6 171.2,39 158.3,14.3 135.6,14.3 159.9,60.5 135.6,106.7 158.3,106.7 171.2,82.1
|
||||
184.2,106.7 191.1,106.7 199,91.7 182.6,60.5 193.5,39.6 "/>
|
||||
<radialGradient id="SVGID_2_" cx="257.2801" cy="60.5308" r="31.5337" gradientTransform="matrix(-1 0 0 1 540.7146 0)" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.1989" style="stop-color:#EE2724"/>
|
||||
<stop offset="0.3065" style="stop-color:#EE2424"/>
|
||||
<stop offset="1" style="stop-color:#CD2027"/>
|
||||
</radialGradient>
|
||||
<polygon fill="url(#SVGID_2_)" points="272.4,18.4 283.3,39 296.3,14.3 319,14.3 294.7,60.5 319,106.7 296.3,106.7 283.3,82.1
|
||||
270.3,106.7 263.5,106.7 255.6,91.7 272,60.5 260.9,39.4 "/>
|
||||
<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="43.4116" y1="15.5743" x2="43.4116" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_3_)" d="M9.3,35h18.8l15.3,27.6L58.7,35h18.8v71.2H59.8V65.3L43.5,93.2h-0.4L26.8,65.5v40.7H9.3V35z"/>
|
||||
<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="108.1806" y1="15.5743" x2="108.1806" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_4_)" d="M99.3,79.5L74.9,35h20.2l13.2,25.9L121.6,35h19.9l-24.4,44.3v27H99.3V79.5z"/>
|
||||
<linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="349.3989" y1="15.5743" x2="349.3989" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_5_)" d="M322.3,35h27.5c4,0,7.7,0.6,11,1.7c3.3,1.1,6.1,2.7,8.4,4.9c2.3,2.1,4.1,4.8,5.4,7.9
|
||||
c1.3,3.1,1.9,6.6,1.9,10.6v0.2c0,4.2-0.7,7.9-2.1,11.1c-1.4,3.2-3.3,5.9-5.7,8c-2.4,2.2-5.4,3.8-8.7,4.9c-3.4,1.1-7.1,1.6-11,1.6
|
||||
h-8.8v20.4h-17.8V35z M348.5,70.4c3.2,0,5.7-0.8,7.5-2.5c1.8-1.7,2.7-3.9,2.7-6.8v-0.2c0-3.1-0.9-5.4-2.7-7
|
||||
c-1.8-1.6-4.3-2.4-7.6-2.4h-8.3v18.9H348.5z"/>
|
||||
<linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="402.2677" y1="15.5743" x2="402.2677" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_6_)" d="M393.7,34.5h17.1l27.3,71.7h-19l-4.7-12.7h-24.7l-4.6,12.7h-18.7L393.7,34.5z M409.3,78.3L402.1,58
|
||||
l-7.2,20.3H409.3z"/>
|
||||
<linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="460.045" y1="15.5743" x2="460.045" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_7_)" d="M462.1,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
|
||||
c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
|
||||
c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9c-2.9-1-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
|
||||
c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
|
||||
c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
|
||||
v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
|
||||
c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
|
||||
c-2.3,2-5,3.5-8.2,4.6C469.6,106.9,466,107.5,462.1,107.5z"/>
|
||||
<linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="512.3254" y1="15.5743" x2="512.3254" y2="380.5575">
|
||||
<stop offset="0" style="stop-color:#FFFFFF"/>
|
||||
<stop offset="1" style="stop-color:#000000"/>
|
||||
</linearGradient>
|
||||
<path fill="url(#SVGID_8_)" d="M514.4,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
|
||||
c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
|
||||
c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9s-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
|
||||
c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
|
||||
c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
|
||||
v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
|
||||
c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
|
||||
c-2.3,2-5,3.5-8.2,4.6C521.8,106.9,518.3,107.5,514.4,107.5z"/>
|
||||
<g>
|
||||
<radialGradient id="SVGID_9_" cx="227.409" cy="56.389" r="44.9787" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.5744" style="stop-color:#E82325"/>
|
||||
<stop offset="1" style="stop-color:#ED1C24"/>
|
||||
</radialGradient>
|
||||
<path fill="url(#SVGID_9_)" d="M215,56.4L188.5,6h24.7l14.1,26.9L241.5,6h24.7l-26.5,50.3l26.5,50.4h-24.7l-14.1-26.9l-14.1,26.9
|
||||
h-24.7L215,56.4z"/>
|
||||
</g>
|
||||
<radialGradient id="SVGID_10_" cx="270.4181" cy="60.5308" r="41.2772" gradientTransform="matrix(-1 0 0 1 556.5228 0)" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3863" style="stop-color:#E81D25"/>
|
||||
<stop offset="0.5804" style="stop-color:#E21E25"/>
|
||||
<stop offset="0.7696" style="stop-color:#D51E25"/>
|
||||
<stop offset="0.9564" style="stop-color:#C12025"/>
|
||||
<stop offset="1" style="stop-color:#BB2025"/>
|
||||
</radialGradient>
|
||||
<polygon fill="url(#SVGID_10_)" points="261.9,14.3 256.2,25.2 256.2,25.2 261.9,14.3 "/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 69 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 187 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
|
|
@ -102,6 +102,7 @@ const networks = [
|
|||
{
|
||||
slug: 'blowpass',
|
||||
name: 'Blowpass',
|
||||
alias: ['myxxxpass'],
|
||||
url: 'https://www.blowpass.com',
|
||||
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
|
||||
parameters: {
|
||||
|
|
@ -435,6 +436,7 @@ exports.seed = knex => Promise.resolve()
|
|||
const networksWithParent = networks.map(network => ({
|
||||
slug: network.slug,
|
||||
name: network.name,
|
||||
alias: (network.alias || []).join(','),
|
||||
url: network.url,
|
||||
description: network.description,
|
||||
parameters: network.parameters,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const logger = require('./logger')(__filename);
|
|||
const slugify = require('./utils/slugify');
|
||||
const capitalize = require('./utils/capitalize');
|
||||
const resolvePlace = require('./utils/resolve-place');
|
||||
const { associateAvatars } = require('./media');
|
||||
|
||||
const { toBaseReleases } = require('./deep');
|
||||
|
||||
|
|
@ -80,6 +81,7 @@ function curateProfileEntry(profile) {
|
|||
has_piercings: profile.hasPiercings,
|
||||
piercings: profile.piercings,
|
||||
tattoos: profile.tattoos,
|
||||
avatar_media_id: profile.avatarMediaId || null,
|
||||
};
|
||||
|
||||
return curatedProfileEntry;
|
||||
|
|
@ -91,6 +93,7 @@ async function curateProfile(profile) {
|
|||
id: profile.id,
|
||||
name: profile.name,
|
||||
avatar: profile.avatar,
|
||||
scraper: profile.scraper,
|
||||
};
|
||||
|
||||
curatedProfile.site = profile.site.isNetwork ? null : profile.site;
|
||||
|
|
@ -198,6 +201,7 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
|
|||
return {
|
||||
...actor,
|
||||
...profile,
|
||||
scraper: scraperSlug,
|
||||
site: siteOrNetwork,
|
||||
};
|
||||
}), Promise.reject(new Error()));
|
||||
|
|
@ -256,16 +260,16 @@ async function upsertProfiles(curatedProfileEntries) {
|
|||
}
|
||||
|
||||
if (argv.force && updatingProfileEntries.length > 0) {
|
||||
knex.transaction(async (transaction) => {
|
||||
const transaction = await knex.transaction();
|
||||
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
|
||||
.where('id', profileEntry.id)
|
||||
.update(profileEntry)
|
||||
.returning(['id', 'actor_id'])
|
||||
.transacting(transaction));
|
||||
|
||||
return Promise.all(queries)
|
||||
await Promise.all(queries)
|
||||
.then(transaction.commit)
|
||||
.catch(transaction.rollback);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -305,7 +309,9 @@ async function scrapeActors(actorNames) {
|
|||
);
|
||||
|
||||
const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
|
||||
const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
|
||||
const profilesWithAvatarIds = await associateAvatars(profiles);
|
||||
|
||||
const curatedProfileEntries = profilesWithAvatarIds.map(profile => curateProfileEntry(profile));
|
||||
|
||||
await upsertProfiles(curatedProfileEntries);
|
||||
}
|
||||
|
|
@ -350,7 +356,7 @@ async function associateActors(releases, batchId) {
|
|||
const baseActors = Object.values(baseActorsByReleaseId).flat();
|
||||
|
||||
if (baseActors.length === 0) {
|
||||
return;
|
||||
return null;
|
||||
}
|
||||
|
||||
const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
|
||||
|
|
@ -382,6 +388,8 @@ async function associateActors(releases, batchId) {
|
|||
.flat();
|
||||
|
||||
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
|
||||
|
||||
return actors;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
52
src/media.js
52
src/media.js
|
|
@ -106,10 +106,11 @@ function toBaseSource(rawSource) {
|
|||
return null;
|
||||
}
|
||||
|
||||
function baseSourceToBaseMedia(baseSource, role) {
|
||||
function baseSourceToBaseMedia(baseSource, role, metadata) {
|
||||
if (Array.isArray(baseSource)) {
|
||||
if (baseSource.length > 0) {
|
||||
return {
|
||||
...metadata,
|
||||
id: nanoid(),
|
||||
role,
|
||||
sources: baseSource,
|
||||
|
|
@ -121,6 +122,7 @@ function baseSourceToBaseMedia(baseSource, role) {
|
|||
|
||||
if (baseSource) {
|
||||
return {
|
||||
...metadata,
|
||||
id: nanoid(),
|
||||
role,
|
||||
sources: [baseSource],
|
||||
|
|
@ -130,15 +132,15 @@ function baseSourceToBaseMedia(baseSource, role) {
|
|||
return null;
|
||||
}
|
||||
|
||||
function fallbackMediaToBaseMedia(rawMedia, role) {
|
||||
function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
|
||||
const baseSources = rawMedia
|
||||
.map(source => toBaseSource(source))
|
||||
.filter(Boolean);
|
||||
|
||||
return baseSourceToBaseMedia(baseSources, role);
|
||||
return baseSourceToBaseMedia(baseSources, role, metadata);
|
||||
}
|
||||
|
||||
function toBaseMedias(rawMedias, role) {
|
||||
function toBaseMedias(rawMedias, role, metadata) {
|
||||
if (!rawMedias || rawMedias.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
|
@ -150,12 +152,12 @@ function toBaseMedias(rawMedias, role) {
|
|||
|
||||
if (Array.isArray(rawMedia)) {
|
||||
// fallback sources provided
|
||||
return fallbackMediaToBaseMedia(rawMedia, role);
|
||||
return fallbackMediaToBaseMedia(rawMedia, role, metadata);
|
||||
}
|
||||
|
||||
const baseSource = toBaseSource(rawMedia);
|
||||
|
||||
return baseSourceToBaseMedia(baseSource, role);
|
||||
return baseSourceToBaseMedia(baseSource, role, metadata);
|
||||
}).filter(Boolean);
|
||||
|
||||
const sampledBaseMedias = sampleMedias(baseMedias);
|
||||
|
|
@ -604,6 +606,44 @@ async function associateReleaseMedia(releases) {
|
|||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function associateAvatars(profiles) {
|
||||
if (!argv.media) {
|
||||
return profiles;
|
||||
}
|
||||
|
||||
const profilesWithBaseMedias = profiles.map(profile => (profile.avatar
|
||||
? {
|
||||
...profile,
|
||||
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
|
||||
copyright: profile.network?.name || null,
|
||||
scraper: profile.scraper || null,
|
||||
})[0],
|
||||
}
|
||||
: profile
|
||||
));
|
||||
|
||||
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
|
||||
|
||||
const storedMedias = await storeMedias(baseMedias);
|
||||
const storedMediasById = itemsByKey(storedMedias, 'id');
|
||||
|
||||
const profilesWithAvatarIds = profilesWithBaseMedias.map((profile) => {
|
||||
const media = storedMediasById[profile.avatarBaseMedia?.id];
|
||||
|
||||
if (media) {
|
||||
return {
|
||||
...profile,
|
||||
avatarMediaId: media.use || media.entry.id,
|
||||
};
|
||||
}
|
||||
|
||||
return profile;
|
||||
});
|
||||
|
||||
return profilesWithAvatarIds;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
associateAvatars,
|
||||
associateReleaseMedia,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -365,8 +365,8 @@ function scrapeApiProfile(data, releases, siteSlug) {
|
|||
if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
|
||||
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
|
||||
|
||||
const avatarPath = Object.values(data.pictures).reverse()[0];
|
||||
if (avatarPath) profile.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
|
||||
const avatarPaths = Object.values(data.pictures).reverse();
|
||||
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
|
||||
|
||||
profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);
|
||||
|
||||
|
|
|
|||
|
|
@ -342,8 +342,6 @@ function scrapeProfile(html, url, actorName) {
|
|||
|
||||
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
|
||||
|
||||
console.log(profile);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ const config = require('config');
|
|||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const slugify = require('./utils/slugify');
|
||||
const { associateActors } = require('./actors');
|
||||
const { associateActors, scrapeActors } = require('./actors');
|
||||
const { associateReleaseTags } = require('./tags');
|
||||
const { curateSite } = require('./sites');
|
||||
const { associateReleaseMedia } = require('./media');
|
||||
|
|
@ -226,18 +226,19 @@ async function storeReleases(releases) {
|
|||
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
|
||||
const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries));
|
||||
|
||||
await Promise.all([
|
||||
const [actors] = await Promise.all([
|
||||
associateActors(releasesWithId, batchId),
|
||||
associateReleaseTags(releasesWithId),
|
||||
]);
|
||||
|
||||
await updateReleasesSearch(releasesWithId.map(release => release.id));
|
||||
|
||||
// media is more error-prone, associate separately
|
||||
await associateReleaseMedia(releasesWithId);
|
||||
await scrapeActors(actors.map(actor => actor.name));
|
||||
|
||||
logger.info(`Stored ${storedReleaseEntries.length} releases`);
|
||||
|
||||
await updateReleasesSearch(releasesWithId.map(release => release.id));
|
||||
|
||||
return releasesWithId;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,17 +36,13 @@ function useProxy(url) {
|
|||
|
||||
const queue = taskQueue();
|
||||
|
||||
queue.on('concurrencyReached:http', () => {
|
||||
logger.silly('Queueing requests');
|
||||
});
|
||||
|
||||
queue.define('http', async ({
|
||||
async function handler({
|
||||
url,
|
||||
method = 'GET',
|
||||
body,
|
||||
headers = {},
|
||||
options = {},
|
||||
}) => {
|
||||
}) {
|
||||
if (body) {
|
||||
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
|
||||
} else {
|
||||
|
|
@ -55,12 +51,12 @@ queue.define('http', async ({
|
|||
|
||||
const reqOptions = {
|
||||
headers: {
|
||||
...(options.defaultHeaders !== false && defaultHeaders),
|
||||
...(options?.defaultHeaders !== false && defaultHeaders),
|
||||
...headers,
|
||||
},
|
||||
...defaultOptions,
|
||||
...options,
|
||||
...(options.timeout && { responseTimeout: options.timeout }),
|
||||
...(options?.timeout && { responseTimeout: options?.timeout }),
|
||||
};
|
||||
|
||||
if (useProxy(url)) {
|
||||
|
|
@ -71,8 +67,8 @@ queue.define('http', async ({
|
|||
? await bhttp[method.toLowerCase()](url, body, reqOptions)
|
||||
: await bhttp[method.toLowerCase()](url, reqOptions);
|
||||
|
||||
if (options.stream && options.destination) {
|
||||
await pipeline(res, ...(options.transforms || []), options.destination);
|
||||
if (options?.stream && options?.destination) {
|
||||
await pipeline(res, ...(options?.transforms || []), options?.destination);
|
||||
}
|
||||
|
||||
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
|
||||
|
|
@ -88,12 +84,22 @@ queue.define('http', async ({
|
|||
code: res.statusCode,
|
||||
status: res.statusCode,
|
||||
};
|
||||
}, {
|
||||
}
|
||||
|
||||
queue.on('concurrencyReached:http', () => {
|
||||
logger.silly('Queueing requests');
|
||||
});
|
||||
|
||||
queue.define('20p', handler, {
|
||||
concurrency: 20,
|
||||
});
|
||||
|
||||
async function get(url, headers, options) {
|
||||
return queue.push('http', {
|
||||
queue.define('1s', handler, {
|
||||
interval: 1,
|
||||
});
|
||||
|
||||
async function get(url, headers, options, queueMethod = '20p') {
|
||||
return queue.push(queueMethod, {
|
||||
method: 'GET',
|
||||
url,
|
||||
headers,
|
||||
|
|
@ -101,8 +107,8 @@ async function get(url, headers, options) {
|
|||
});
|
||||
}
|
||||
|
||||
async function post(url, body, headers, options) {
|
||||
return queue.push('http', {
|
||||
async function post(url, body, headers, options, queueMethod = '20p') {
|
||||
return queue.push(queueMethod, {
|
||||
method: 'POST',
|
||||
url,
|
||||
body,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ async function resolvePlace(query) {
|
|||
// https://operations.osmfoundation.org/policies/nominatim/
|
||||
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
|
||||
'User-Agent': 'contact at moonloop.adult@protonmail.com',
|
||||
});
|
||||
}, null, '1s');
|
||||
|
||||
const [item] = res.body;
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
|
|||
}
|
||||
|
||||
extend type Actor {
|
||||
age: Int @requires(columns: ["birthdate"])
|
||||
age: Int @requires(columns: ["date_of_birth"])
|
||||
height(units:Units): String @requires(columns: ["height"])
|
||||
weight(units:Units): String @requires(columns: ["weight"])
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue