diff --git a/assets/components/actors/tile.vue b/assets/components/actors/tile.vue index 0c7f3af5..5c0a45d7 100644 --- a/assets/components/actors/tile.vue +++ b/assets/components/actors/tile.vue @@ -16,13 +16,13 @@ >{{ actor.name }} diff --git a/assets/js/fragments.js b/assets/js/fragments.js index 197a555d..bc95747a 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -61,7 +61,7 @@ const actorFields = ` lazy } } - network: entity { + entity { id name slug diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 405c2887..ab5d53a9 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -682,6 +682,11 @@ exports.up = knex => Promise.resolve() .inTable('actors') .onDelete('cascade'); + table.integer('alias_id', 12) + .references('id') + .inTable('actors') + .onDelete('cascade'); + table.unique(['release_id', 'actor_id']); table.datetime('created_at') diff --git a/package-lock.json b/package-lock.json index 1b7bf874..8581ea5d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "traxxx", - "version": "1.171.0", + "version": "1.172.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 1c9a707c..968250a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "traxxx", - "version": "1.171.0", + "version": "1.172.0", "description": "All the latest porn releases in one place", "main": "src/app.js", "scripts": { diff --git a/public/img/logos/nebraskacoeds/favicon.png b/public/img/logos/nebraskacoeds/favicon.png new file mode 100644 index 00000000..ff3f0bbb Binary files /dev/null and b/public/img/logos/nebraskacoeds/favicon.png differ diff --git a/src/actors.js b/src/actors.js index 474e1e56..3f5146cc 100644 --- a/src/actors.js +++ b/src/actors.js @@ -180,6 +180,11 @@ function toBaseActors(actorsOrNames, release) { return baseActors; } +function getCollisionLikely(actor) { + // actor with single name + return actor.name.match(/\w+/g).length === 1; +} + function curateActor(actor, withDetails = false, isProfile = false) { if (!actor) { return null; @@ -260,11 +265,13 @@ function curateActor(actor, withDetails = false, isProfile = false) { } function curateActorEntry(baseActor, batchId) { + const collisionLikely = getCollisionLikely(baseActor); + return { name: baseActor.name, slug: baseActor.slug, - entity_id: null, - entry_id: baseActor.entryId, + entity_id: collisionLikely ? baseActor.entity.id : null, + entry_id: collisionLikely ? baseActor.entryId : null, batch_id: batchId, }; } @@ -641,6 +648,11 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy const scraper = scrapers[scraperSlug]; const layoutScraper = resolveLayoutScraper(entity, scraper); + if (!layoutScraper?.fetchProfile) { + logger.warn(`No profile profile scraper available for ${scraperSlug}`); + throw new Error(`No profile profile scraper available for ${scraperSlug}`); + } + const context = { ...entity, // legacy @@ -653,11 +665,6 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy const label = context.entity?.name; - if (!layoutScraper?.fetchProfile) { - logger.warn(`No profile profile scraper available for ${scraperSlug}`); - throw new Error(`No profile profile scraper available for ${scraperSlug}`); - } - if (!context.entity) { logger.warn(`No entity found for ${scraperSlug}`); throw new Error(`No entity found for ${scraperSlug}`); @@ -813,33 +820,53 @@ async function scrapeActors(argNames) { async function getOrCreateActors(baseActors, batchId) { // WHERE IN causes stack depth error and performance issues with a large amount of values, no knex VALUES helper available - const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId)', { slug: actor.slug, entityId: actor.entity.id })).join(', '); + const actorValues = baseActors.map(actor => knex.raw('(:slug, :entityId, :entryId, :collisionLikely)', { + slug: actor.slug, + entityId: actor.entity.id, + entryId: actor.entryId, + collisionLikely: getCollisionLikely(actor), + })).join(', '); const existingActors = await knex .select('actors.*') - .from(knex.raw(`actors, (VALUES ${actorValues}) AS base_actors (slug, entity_id)`)) - .whereRaw('actors.slug = base_actors.slug AND actors.entity_id IS NULL') - .orWhereRaw('actors.slug = base_actors.slug AND actors.entity_id = base_actors.entity_id'); + .from(knex.raw(`actors, (VALUES ${actorValues}) AS base_actors (slug, entity_id, entry_id, collision_likely)`)) + .whereRaw(` + actors.slug = base_actors.slug + AND actors.entity_id IS NULL + AND NOT base_actors.collision_likely + `) + .orWhereRaw(` + actors.slug = base_actors.slug + AND actors.entity_id = base_actors.entity_id + AND ((actors.entry_id IS NULL AND base_actors.entry_id IS NULL) + OR actors.entry_id = base_actors.entry_id) + `); // const existingActorSlugs = new Set(existingActors.map(actor => actor.slug)); const existingActorSlugs = existingActors.reduce((acc, actor) => ({ ...acc, [actor.entity_id]: { ...acc[actor.entity_id], - [actor.slug]: true, + [actor.entry_id]: { + ...acc[actor.entity_id]?.[actor.entry_id], + [actor.slug]: true, + }, }, }), {}); - const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]); - + const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.entryId]?.[baseActor.slug] && !existingActorSlugs.null?.null?.[baseActor.slug]); const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId); + const newActors = await bulkInsert('actors', curatedActorEntries); - const newActorIdsByEntityIdAndSlug = newActors.reduce((acc, actor) => ({ + const newActorIdsByEntityIdEntryIdAndSlug = newActors.reduce((acc, actor) => ({ ...acc, [actor.entity_id]: { ...acc[actor.entity_id], - [actor.slug]: actor.id, + [actor.entry_id]: { + ...acc[actor.entity_id]?.[actor.entry_id], + [actor.slug]: actor.id, + }, }, }), {}); @@ -847,7 +874,7 @@ async function getOrCreateActors(baseActors, batchId) { .filter(actor => actor.hasProfile) .map(actor => ({ ...actor, - id: newActorIdsByEntityIdAndSlug[actor.entity?.id]?.[actor.slug] || newActorIdsByEntityIdAndSlug.null?.[actor.slug], + id: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug], })) .filter(actor => !!actor.id) .map(actor => curateProfile(actor))); @@ -885,16 +912,32 @@ async function associateActors(releases, batchId) { const actors = await getOrCreateActors(uniqueBaseActors, batchId); + /* const actorIdsBySlug = actors.reduce((acc, actor) => ({ ...acc, [actor.slug]: actor.alias_for || actor.id, }), {}); + */ + + const actorIdsByEntityIdEntryIdAndSlug = actors.reduce((acc, actor) => ({ + ...acc, + [actor.entity_id]: { + ...acc[actor.entity_id], + [actor.entry_id]: { + ...acc[actor.entity_id]?.[actor.entry_id], + [actor.slug]: { + actor_id: actor.alias_for || actor.id, + alias_id: actor.alias_for ? actor.id : null, + }, + }, + }, + }), {}); const releaseActorAssociations = Object.entries(baseActorsByReleaseId) .map(([releaseId, releaseActors]) => releaseActors .map(releaseActor => ({ release_id: releaseId, - actor_id: actorIdsBySlug[releaseActor.slug], + ...(actorIdsByEntityIdEntryIdAndSlug[releaseActor.entity?.id]?.[releaseActor.entryId]?.[releaseActor.slug] || actorIdsByEntityIdEntryIdAndSlug.null.null[releaseActor.slug]), }))) .flat(); diff --git a/src/scrapers/elevatedx.js b/src/scrapers/elevatedx.js index 39cdc709..c7b39939 100644 --- a/src/scrapers/elevatedx.js +++ b/src/scrapers/elevatedx.js @@ -66,7 +66,7 @@ function scrapeAllClassic(scenes, channel) { release.title = query.cnt('.updateInfo h5 a'); release.actors = query.cnts('.tour_update_models a'); - release.date = query.date('.availdate, .updateInfo p span:nth-child(2)', 'MM/DD/YYYY'); + release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY'); release.poster = query.img('.updateThumb img'); diff --git a/src/scrapers/porncz.js b/src/scrapers/porncz.js index 1b522b12..6513b479 100644 --- a/src/scrapers/porncz.js +++ b/src/scrapers/porncz.js @@ -47,7 +47,7 @@ function scrapeScene({ query }, url, channel) { release.poster = query.img('#video-poster', 'data-poster', { origin: channel.url }); release.photos = query.imgs('#gallery .photo-item img', 'data-src', { origin: channel.url }); - release.trailer = query.video('.trailer source'); + release.trailer = query.video(); release.channel = slugify(query.q('.video-detail-logo img', 'alt'), ''); diff --git a/src/scrapers/resolve.js b/src/scrapers/resolve.js index 89726b56..db72f51a 100644 --- a/src/scrapers/resolve.js +++ b/src/scrapers/resolve.js @@ -19,7 +19,7 @@ function resolveLayoutScraper(entity, scraper) { return scraper[entity.parameters.layout]; } - if (entity.parent) { + if (entity?.parent) { return resolveLayoutScraper(entity.parent, scraper); } diff --git a/src/scrapers/traxxx.js b/src/scrapers/traxxx.js index 76d51060..3016f1ac 100644 --- a/src/scrapers/traxxx.js +++ b/src/scrapers/traxxx.js @@ -222,7 +222,11 @@ function actors(release) { : Math.floor(Math.random() * 3) + 2; return Array.from({ length }, () => ({ - name: faker.name.findName(), + name: faker.name + .findName() + .split(' ') + .slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name + .join(' '), gender: gender(), })); }