Storing actor avatars. Using 1 second interval queue for location resolve as per OSM code of conduct.

This commit is contained in:
ThePendulum 2020-05-16 04:36:45 +02:00
parent 21d4dd6bfa
commit 05ee57378a
28 changed files with 899 additions and 751 deletions

View File

@ -36,18 +36,15 @@ function curateActor(actor) {
updatedAt: new Date(actor.updatedAt),
};
if (actor.avatar) {
curatedActor.avatar = actor.avatar.media;
if (actor.profiles && actor.profiles.length > 0) {
curatedActor.avatar = actor.profiles.slice(0, 1)[0].avatar;
curatedActor.photos = actor.profiles.slice(1).map(profile => profile.avatar);
}
if (actor.releases) {
curatedActor.releases = actor.releases.map(release => curateRelease(release.release));
}
if (actor.photos) {
curatedActor.photos = actor.photos.map(photo => photo.media);
}
return curatedActor;
}
@ -99,21 +96,14 @@ function initActorActions(store, _router) {
name
slug
}
actorsProfiles {
actorsAvatarByProfileId {
media {
path
thumbnail
copyright
}
}
}
photos: actorsPhotos {
media {
profiles: actorsProfiles {
description
avatar: avatarMedia {
id
thumbnail
path
index
thumbnail
lazy
comment
copyright
}
}
@ -239,6 +229,7 @@ function initActorActions(store, _router) {
actorsProfiles {
actorsAvatarByProfileId {
media {
id
path
thumbnail
copyright

View File

@ -59,13 +59,16 @@ exports.up = knex => Promise.resolve()
.inTable('media')
.unique();
}))
.then(() => knex.raw(`
.then(() => { // eslint-disable-line arrow-body-style
// allow vim fold
return knex.raw(`
CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
SELECT media_id FROM media_sfw
ORDER BY random()
LIMIT 1;
$$ LANGUAGE sql STABLE;
`))
`);
})
.then(() => knex.schema.alterTable('media', (table) => {
table.string('sfw_media_id', 21)
.references('id')
@ -140,6 +143,7 @@ exports.up = knex => Promise.resolve()
table.increments('id', 12);
table.string('name');
table.string('alias');
table.string('url');
table.text('description');
table.json('parameters');
@ -322,6 +326,7 @@ exports.up = knex => Promise.resolve()
table.increments('id', 12);
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
@ -371,6 +376,10 @@ exports.up = knex => Promise.resolve()
table.string('piercings');
table.string('tattoos');
table.string('avatar_media_id', 21)
.references('id')
.inTable('media');
table.datetime('scraped_at');
table.boolean('scrape_success');
@ -386,7 +395,9 @@ exports.up = knex => Promise.resolve()
table.string('name');
}))
.then(() => knex('body').insert([
.then(() => { // eslint-disable-line arrow-body-style
// allow vim fold
return knex('body').insert([
// head
{ slug: 'head', name: 'head' },
{ slug: 'face', name: 'face' },
@ -501,7 +512,8 @@ exports.up = knex => Promise.resolve()
{ slug: 'middle-toe', name: 'middle toe' },
{ slug: 'fourth-toe', name: 'fourth toe' },
{ slug: 'little-toe', name: 'little toe' },
]))
]);
})
.then(() => knex.schema.createTable('actors_tattoos', (table) => {
table.increments('id');
@ -769,7 +781,9 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('releases');
}))
.then(() => knex.raw(`
.then(() => { // eslint-disable-line arrow-body-style
// allow vim fold
return knex.raw(`
ALTER TABLE releases
ADD CONSTRAINT ensure_site_or_network CHECK (site_id IS NOT NULL OR network_id IS NOT NULL);
@ -828,9 +842,12 @@ exports.up = knex => Promise.resolve()
COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many';
COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many';
`));
`);
});
exports.down = knex => knex.raw(`
exports.down = (knex) => { // eslint-disable-line arrow-body-style
// allow vim fold
return knex.raw(`
DROP VIEW IF EXISTS movie_actors;
DROP VIEW IF EXISTS movie_tags;
@ -880,3 +897,4 @@ exports.down = knex => knex.raw(`
DROP TEXT SEARCH CONFIGURATION IF EXISTS traxxx;
DROP TEXT SEARCH DICTIONARY IF EXISTS traxxx_dict;
`);
};

Binary file not shown.

After

Width:  |  Height:  |  Size: 81 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 140 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

View File

@ -0,0 +1,84 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 547 112" enable-background="new 0 0 547 112" xml:space="preserve">
<radialGradient id="SVGID_1_" cx="171.1188" cy="60.5308" r="34.3229" gradientUnits="userSpaceOnUse">
<stop offset="0.1989" style="stop-color:#EE2724"/>
<stop offset="0.3065" style="stop-color:#EE2424"/>
<stop offset="1" style="stop-color:#CD2027"/>
</radialGradient>
<polygon fill="url(#SVGID_1_)" points="182.1,18.6 171.2,39 158.3,14.3 135.6,14.3 159.9,60.5 135.6,106.7 158.3,106.7 171.2,82.1
184.2,106.7 191.1,106.7 199,91.7 182.6,60.5 193.5,39.6 "/>
<radialGradient id="SVGID_2_" cx="257.2801" cy="60.5308" r="31.5337" gradientTransform="matrix(-1 0 0 1 540.7146 0)" gradientUnits="userSpaceOnUse">
<stop offset="0.1989" style="stop-color:#EE2724"/>
<stop offset="0.3065" style="stop-color:#EE2424"/>
<stop offset="1" style="stop-color:#CD2027"/>
</radialGradient>
<polygon fill="url(#SVGID_2_)" points="272.4,18.4 283.3,39 296.3,14.3 319,14.3 294.7,60.5 319,106.7 296.3,106.7 283.3,82.1
270.3,106.7 263.5,106.7 255.6,91.7 272,60.5 260.9,39.4 "/>
<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="43.4116" y1="15.5743" x2="43.4116" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_3_)" d="M9.3,35h18.8l15.3,27.6L58.7,35h18.8v71.2H59.8V65.3L43.5,93.2h-0.4L26.8,65.5v40.7H9.3V35z"/>
<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="108.1806" y1="15.5743" x2="108.1806" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_4_)" d="M99.3,79.5L74.9,35h20.2l13.2,25.9L121.6,35h19.9l-24.4,44.3v27H99.3V79.5z"/>
<linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="349.3989" y1="15.5743" x2="349.3989" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_5_)" d="M322.3,35h27.5c4,0,7.7,0.6,11,1.7c3.3,1.1,6.1,2.7,8.4,4.9c2.3,2.1,4.1,4.8,5.4,7.9
c1.3,3.1,1.9,6.6,1.9,10.6v0.2c0,4.2-0.7,7.9-2.1,11.1c-1.4,3.2-3.3,5.9-5.7,8c-2.4,2.2-5.4,3.8-8.7,4.9c-3.4,1.1-7.1,1.6-11,1.6
h-8.8v20.4h-17.8V35z M348.5,70.4c3.2,0,5.7-0.8,7.5-2.5c1.8-1.7,2.7-3.9,2.7-6.8v-0.2c0-3.1-0.9-5.4-2.7-7
c-1.8-1.6-4.3-2.4-7.6-2.4h-8.3v18.9H348.5z"/>
<linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="402.2677" y1="15.5743" x2="402.2677" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_6_)" d="M393.7,34.5h17.1l27.3,71.7h-19l-4.7-12.7h-24.7l-4.6,12.7h-18.7L393.7,34.5z M409.3,78.3L402.1,58
l-7.2,20.3H409.3z"/>
<linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="460.045" y1="15.5743" x2="460.045" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_7_)" d="M462.1,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9c-2.9-1-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
c-2.3,2-5,3.5-8.2,4.6C469.6,106.9,466,107.5,462.1,107.5z"/>
<linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="512.3254" y1="15.5743" x2="512.3254" y2="380.5575">
<stop offset="0" style="stop-color:#FFFFFF"/>
<stop offset="1" style="stop-color:#000000"/>
</linearGradient>
<path fill="url(#SVGID_8_)" d="M514.4,107.5c-5.7,0-11.2-1-16.3-3c-5.2-2-9.7-5-13.7-8.9l9.9-13.2c6.4,5.7,13.3,8.5,20.8,8.5
c2.4,0,4.2-0.4,5.4-1.3c1.3-0.8,1.9-2,1.9-3.6v-0.2c0-0.7-0.2-1.4-0.5-2c-0.3-0.6-0.9-1.1-1.7-1.7c-0.8-0.5-1.9-1.1-3.3-1.6
c-1.4-0.5-3.2-1-5.4-1.6c-3.4-0.9-6.6-1.8-9.5-2.9s-5.5-2.4-7.6-4.1c-2.2-1.7-3.9-3.7-5.1-6.2c-1.3-2.5-1.9-5.5-1.9-9.2v-0.2
c0-3.3,0.6-6.4,1.7-9.1c1.1-2.8,2.8-5.2,4.9-7.2c2.2-2,4.8-3.6,7.8-4.7c3.1-1.1,6.5-1.7,10.3-1.7c5.5,0,10.4,0.8,14.7,2.4
c4.3,1.6,8.2,4,11.7,7.1l-8.9,14c-2.9-2.3-5.9-4.1-9-5.2c-3.1-1.2-6.1-1.8-8.9-1.8c-2.1,0-3.7,0.4-4.8,1.3c-1.1,0.9-1.6,2-1.6,3.2
v0.2c0,0.8,0.2,1.5,0.5,2.1c0.3,0.6,0.9,1.2,1.7,1.7c0.8,0.5,2,1,3.4,1.5c1.4,0.5,3.3,1,5.4,1.6c3.7,0.9,7,1.9,9.9,3.1
c3,1.2,5.5,2.6,7.6,4.3c2.1,1.7,3.7,3.7,4.8,6.1s1.6,5.3,1.6,8.6v0.2c0,3.7-0.6,6.9-1.9,9.8c-1.3,2.8-3,5.3-5.3,7.3
c-2.3,2-5,3.5-8.2,4.6C521.8,106.9,518.3,107.5,514.4,107.5z"/>
<g>
<radialGradient id="SVGID_9_" cx="227.409" cy="56.389" r="44.9787" gradientUnits="userSpaceOnUse">
<stop offset="0.5744" style="stop-color:#E82325"/>
<stop offset="1" style="stop-color:#ED1C24"/>
</radialGradient>
<path fill="url(#SVGID_9_)" d="M215,56.4L188.5,6h24.7l14.1,26.9L241.5,6h24.7l-26.5,50.3l26.5,50.4h-24.7l-14.1-26.9l-14.1,26.9
h-24.7L215,56.4z"/>
</g>
<radialGradient id="SVGID_10_" cx="270.4181" cy="60.5308" r="41.2772" gradientTransform="matrix(-1 0 0 1 556.5228 0)" gradientUnits="userSpaceOnUse">
<stop offset="0.3863" style="stop-color:#E81D25"/>
<stop offset="0.5804" style="stop-color:#E21E25"/>
<stop offset="0.7696" style="stop-color:#D51E25"/>
<stop offset="0.9564" style="stop-color:#C12025"/>
<stop offset="1" style="stop-color:#BB2025"/>
</radialGradient>
<polygon fill="url(#SVGID_10_)" points="261.9,14.3 256.2,25.2 256.2,25.2 261.9,14.3 "/>
</svg>

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 70 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 187 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

View File

@ -102,6 +102,7 @@ const networks = [
{
slug: 'blowpass',
name: 'Blowpass',
alias: ['myxxxpass'],
url: 'https://www.blowpass.com',
description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
parameters: {
@ -435,6 +436,7 @@ exports.seed = knex => Promise.resolve()
const networksWithParent = networks.map(network => ({
slug: network.slug,
name: network.name,
alias: (network.alias || []).join(','),
url: network.url,
description: network.description,
parameters: network.parameters,

View File

@ -13,6 +13,7 @@ const logger = require('./logger')(__filename);
const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize');
const resolvePlace = require('./utils/resolve-place');
const { associateAvatars } = require('./media');
const { toBaseReleases } = require('./deep');
@ -80,6 +81,7 @@ function curateProfileEntry(profile) {
has_piercings: profile.hasPiercings,
piercings: profile.piercings,
tattoos: profile.tattoos,
avatar_media_id: profile.avatarMediaId || null,
};
return curatedProfileEntry;
@ -91,6 +93,7 @@ async function curateProfile(profile) {
id: profile.id,
name: profile.name,
avatar: profile.avatar,
scraper: profile.scraper,
};
curatedProfile.site = profile.site.isNetwork ? null : profile.site;
@ -198,6 +201,7 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
return {
...actor,
...profile,
scraper: scraperSlug,
site: siteOrNetwork,
};
}), Promise.reject(new Error()));
@ -256,16 +260,16 @@ async function upsertProfiles(curatedProfileEntries) {
}
if (argv.force && updatingProfileEntries.length > 0) {
knex.transaction(async (transaction) => {
const transaction = await knex.transaction();
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
.where('id', profileEntry.id)
.update(profileEntry)
.returning(['id', 'actor_id'])
.transacting(transaction));
return Promise.all(queries)
await Promise.all(queries)
.then(transaction.commit)
.catch(transaction.rollback);
});
}
}
@ -305,7 +309,9 @@ async function scrapeActors(actorNames) {
);
const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
const profilesWithAvatarIds = await associateAvatars(profiles);
const curatedProfileEntries = profilesWithAvatarIds.map(profile => curateProfileEntry(profile));
await upsertProfiles(curatedProfileEntries);
}
@ -350,7 +356,7 @@ async function associateActors(releases, batchId) {
const baseActors = Object.values(baseActorsByReleaseId).flat();
if (baseActors.length === 0) {
return;
return null;
}
const baseActorsBySlugAndNetworkId = baseActors.reduce((acc, baseActor) => ({
@ -382,6 +388,8 @@ async function associateActors(releases, batchId) {
.flat();
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
return actors;
}
module.exports = {

View File

@ -106,10 +106,11 @@ function toBaseSource(rawSource) {
return null;
}
function baseSourceToBaseMedia(baseSource, role) {
function baseSourceToBaseMedia(baseSource, role, metadata) {
if (Array.isArray(baseSource)) {
if (baseSource.length > 0) {
return {
...metadata,
id: nanoid(),
role,
sources: baseSource,
@ -121,6 +122,7 @@ function baseSourceToBaseMedia(baseSource, role) {
if (baseSource) {
return {
...metadata,
id: nanoid(),
role,
sources: [baseSource],
@ -130,15 +132,15 @@ function baseSourceToBaseMedia(baseSource, role) {
return null;
}
function fallbackMediaToBaseMedia(rawMedia, role) {
function fallbackMediaToBaseMedia(rawMedia, role, metadata) {
const baseSources = rawMedia
.map(source => toBaseSource(source))
.filter(Boolean);
return baseSourceToBaseMedia(baseSources, role);
return baseSourceToBaseMedia(baseSources, role, metadata);
}
function toBaseMedias(rawMedias, role) {
function toBaseMedias(rawMedias, role, metadata) {
if (!rawMedias || rawMedias.length === 0) {
return [];
}
@ -150,12 +152,12 @@ function toBaseMedias(rawMedias, role) {
if (Array.isArray(rawMedia)) {
// fallback sources provided
return fallbackMediaToBaseMedia(rawMedia, role);
return fallbackMediaToBaseMedia(rawMedia, role, metadata);
}
const baseSource = toBaseSource(rawMedia);
return baseSourceToBaseMedia(baseSource, role);
return baseSourceToBaseMedia(baseSource, role, metadata);
}).filter(Boolean);
const sampledBaseMedias = sampleMedias(baseMedias);
@ -604,6 +606,44 @@ async function associateReleaseMedia(releases) {
}, Promise.resolve());
}
async function associateAvatars(profiles) {
if (!argv.media) {
return profiles;
}
const profilesWithBaseMedias = profiles.map(profile => (profile.avatar
? {
...profile,
avatarBaseMedia: toBaseMedias([profile.avatar], 'avatars', {
copyright: profile.network?.name || null,
scraper: profile.scraper || null,
})[0],
}
: profile
));
const baseMedias = profilesWithBaseMedias.map(profile => profile.avatarBaseMedia).filter(Boolean);
const storedMedias = await storeMedias(baseMedias);
const storedMediasById = itemsByKey(storedMedias, 'id');
const profilesWithAvatarIds = profilesWithBaseMedias.map((profile) => {
const media = storedMediasById[profile.avatarBaseMedia?.id];
if (media) {
return {
...profile,
avatarMediaId: media.use || media.entry.id,
};
}
return profile;
});
return profilesWithAvatarIds;
}
module.exports = {
associateAvatars,
associateReleaseMedia,
};

View File

@ -365,8 +365,8 @@ function scrapeApiProfile(data, releases, siteSlug) {
if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
const avatarPath = Object.values(data.pictures).reverse()[0];
if (avatarPath) profile.avatar = `https://images01-evilangel.gammacdn.com/actors${avatarPath}`;
const avatarPaths = Object.values(data.pictures).reverse();
if (avatarPaths.length > 0) profile.avatar = avatarPaths.map(avatarPath => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
profile.releases = releases.map(release => `https://${siteSlug}.com/en/video/${release.url_title}/${release.clip_id}`);

View File

@ -342,8 +342,6 @@ function scrapeProfile(html, url, actorName) {
profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), el => el.href);
console.log(profile);
return profile;
}

View File

@ -5,7 +5,7 @@ const config = require('config');
const logger = require('./logger')(__filename);
const knex = require('./knex');
const slugify = require('./utils/slugify');
const { associateActors } = require('./actors');
const { associateActors, scrapeActors } = require('./actors');
const { associateReleaseTags } = require('./tags');
const { curateSite } = require('./sites');
const { associateReleaseMedia } = require('./media');
@ -226,18 +226,19 @@ async function storeReleases(releases) {
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries));
await Promise.all([
const [actors] = await Promise.all([
associateActors(releasesWithId, batchId),
associateReleaseTags(releasesWithId),
]);
await updateReleasesSearch(releasesWithId.map(release => release.id));
// media is more error-prone, associate separately
await associateReleaseMedia(releasesWithId);
await scrapeActors(actors.map(actor => actor.name));
logger.info(`Stored ${storedReleaseEntries.length} releases`);
await updateReleasesSearch(releasesWithId.map(release => release.id));
return releasesWithId;
}

View File

@ -36,17 +36,13 @@ function useProxy(url) {
const queue = taskQueue();
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
});
queue.define('http', async ({
async function handler({
url,
method = 'GET',
body,
headers = {},
options = {},
}) => {
}) {
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
@ -55,12 +51,12 @@ queue.define('http', async ({
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...(options?.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
...(options?.timeout && { responseTimeout: options?.timeout }),
};
if (useProxy(url)) {
@ -71,8 +67,8 @@ queue.define('http', async ({
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
if (options?.stream && options?.destination) {
await pipeline(res, ...(options?.transforms || []), options?.destination);
}
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
@ -88,12 +84,22 @@ queue.define('http', async ({
code: res.statusCode,
status: res.statusCode,
};
}, {
}
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
});
queue.define('20p', handler, {
concurrency: 20,
});
async function get(url, headers, options) {
return queue.push('http', {
queue.define('1s', handler, {
interval: 1,
});
async function get(url, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
method: 'GET',
url,
headers,
@ -101,8 +107,8 @@ async function get(url, headers, options) {
});
}
async function post(url, body, headers, options) {
return queue.push('http', {
async function post(url, body, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
method: 'POST',
url,
body,

View File

@ -12,7 +12,7 @@ async function resolvePlace(query) {
// https://operations.osmfoundation.org/policies/nominatim/
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
'User-Agent': 'contact at moonloop.adult@protonmail.com',
});
}, null, '1s');
const [item] = res.body;

View File

@ -12,7 +12,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({
}
extend type Actor {
age: Int @requires(columns: ["birthdate"])
age: Int @requires(columns: ["date_of_birth"])
height(units:Units): String @requires(columns: ["height"])
weight(units:Units): String @requires(columns: ["weight"])
}