Added various tag photos. Renamed some toy tags.

assets/components/tags/tags.vue

@@ -65,30 +65,37 @@ async function mounted() {
 		],
 		oral: [
 			'blowjob',
-			'deepthroat',
-			'facefucking',
-			'double-blowjob',
-			'blowbang',
 			'pussy-eating',
 			'ass-eating',
+			'69',
+			'double-blowjob',
+			'deepthroat',
+			'facefucking',
+			'blowbang',
 			'atm',
 		],
+		manual: [
+			'handjob',
+			'fingering',
+			'anal-fingering',
+			'titty-fucking',
+		],
 		cumshot: [
 			'facial',
-			'bukkake',
+			'cum-on-boobs',
+			'cum-on-butt',
+			'cum-in-mouth',
 			'creampie',
 			'anal-creampie',
-			'cum-in-mouth',
 			'oral-creampie',
-			'cum-on-butt',
+			'bukkake',
 		],
-		extreme: [
-			'airtight',
-			'dap',
-			'dvp',
-			'da-tp',
-			'dv-tp',
-			'tap',
+		toys: [
+			'toy-anal',
+			'toy-dp',
+			'double-dildo',
+			'double-dildo-blowjob',
+			'double-dildo-anal',
 		],
 		roleplay: [
 			'family',
@@ -99,13 +106,17 @@ async function mounted() {
 		],
 		fetish: [
 			'bdsm',
-			'bondage',
 			'femdom',
+			'bondage',
+			'blindfold',
 		],
-		toys: [
-			'anal-toys',
-			'double-dildo',
-			'double-dildo-blowjob',
+		extreme: [
+			'airtight',
+			'dap',
+			'dvp',
+			'da-tp',
+			'dv-tp',
+			'tap',
 		],
 		misc: [
 			'gaping',
@@ -161,6 +172,12 @@ export default {
 
 .heading {
     text-transform: capitalize;
+	padding: 0 0 0 .5rem;
+	margin: 2rem 0 1rem 0;
+}
+
+.category:first-child .heading {
+	margin: .5rem 0 1rem 0;
 }
 
 @media(max-width: $breakpoint3) {

seeds/00_tags.js

@@ -103,8 +103,8 @@ const tags = [
 		group: 'penetration',
 	},
 	{
-		name: 'ass fingering',
-		slug: 'ass-fingering',
+		name: 'anal fingering',
+		slug: 'anal-fingering',
 		description: 'Inserting one or multiple fingers into the asshole.',
 	},
 	{
@@ -116,12 +116,6 @@ const tags = [
 		name: 'anal prolapse',
 		slug: 'anal-prolapse',
 	},
-	{
-		name: 'anal toys',
-		slug: 'anal-toys',
-		description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
-		priority: 7,
-	},
 	{
 		name: 'animated',
 		slug: 'animated',
@@ -212,6 +206,11 @@ const tags = [
 		slug: 'black-hair',
 		group: 'hair',
 	},
+	{
+		name: 'blindfold',
+		slug: 'blindfold',
+		group: 'clothing',
+	},
 	{
 		name: 'blonde',
 		slug: 'blonde',
@@ -424,7 +423,7 @@ const tags = [
 		group: 'finish',
 	},
 	{
-		name: 'family taboo',
+		name: 'family',
 		slug: 'family',
 		priority: 7,
 	},
@@ -851,14 +850,26 @@ const tags = [
 		group: 'age',
 	},
 	{
-		name: 'titty fuck',
-		slug: 'titty-fuck',
+		name: 'titty fucking',
+		slug: 'titty-fucking',
 	},
 	{
 		name: 'toys',
 		slug: 'toys',
 		priority: 7,
 	},
+	{
+		name: 'toy anal',
+		slug: 'toy-anal',
+		description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
+		priority: 7,
+	},
+	{
+		name: 'toy DP',
+		slug: 'toy-dp',
+		description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs.',
+		priority: 4,
+	},
 	{
 		name: 'transsexual',
 		slug: 'transsexual',
@@ -987,8 +998,12 @@ const aliases = [
 		for: 'asian',
 	},
 	{
-		name: 'anal fingering',
-		for: 'ass-fingering',
+		name: 'ass fingering',
+		for: 'anal-fingering',
+	},
+	{
+		name: 'anal toys',
+		for: 'toy-anal',
 	},
 	{
 		name: 'ass licking',
@@ -1002,6 +1017,11 @@ const aliases = [
 		name: 'ass fucking',
 		for: 'anal',
 	},
+	{
+		name: 'ass to ass',
+		for: 'double-dildo-anal',
+		secondary: true,
+	},
 	{
 		name: 'ass to other girl\'s mouth',
 		for: 'atogm',
@@ -1050,13 +1070,18 @@ const aliases = [
 	},
 	{
 		name: 'boob fucking',
-		for: 'titty-fuck',
+		for: 'titty-fucking',
 	},
 	{
 		name: 'behind the scenes',
 		for: 'bts',
 		secondary: true,
 	},
+	{
+		name: 'blindfolded',
+		for: 'blindfold',
+		secondary: true,
+	},
 	{
 		name: 'making of',
 		for: 'bts',
@@ -1711,7 +1736,7 @@ const aliases = [
 	},
 	{
 		name: 'tittyfuck',
-		for: 'titty-fuck',
+		for: 'titty-fucking',
 		secondary: true,
 	},
 	{

seeds/02_sites.js

@@ -4246,7 +4246,7 @@ const sites = [
 		name: 'Operación Limpieza',
 		slug: 'operacionlimpieza',
 		url: 'https://mamacitaz.com/channels/operation-limpieza.en.html',
-		tags: ['latina', 'maid'],
+		tags: ['latina'],
 		parent: 'mamacitaz',
 	},
 	// METRO HD

seeds/04_media.js

@@ -586,17 +586,17 @@ const sfw = Object.entries({
 	.flat();
 
 const tagPosters = [
-	['69', 1, 'Melody Petite for 21Sextury'],
+	['69', 0, 'Abby Lee Brazil and Ramon Nomar for Wicked'],
 	['airtight', 6, 'Remy Lacroix in "Ass Worship 14" for Jules Jordan'],
 	['anal', 0, 'Adriana Chechik in "Manuel Creampies Their Asses 3" for Jules Jordan'],
 	['anal-creampie', 1, 'Aleska Diamond in "Aleska Wants More" for Asshole Fever'],
-	['anal-toys', 0, 'Kira Noir in 1225 for InTheCrack'],
-	['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'],
+	['ass-eating', 1, 'Aidra Fox and Cassidy Klein in "Lesbian Anal Yoga" for LesbianX'],
 	['asian', 0, 'Vina Sky for Erotica X'],
 	['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],
 	['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
 	['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
 	['bts', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
+	['blindfold', 0, 'Kylie Page in "Natural Blindfolded Beauties" for Hustler'],
 	['blonde', 1, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
 	['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'],
 	['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
@@ -607,11 +607,13 @@ const tagPosters = [
 	['creampie', 'poster', 'ALina Lopez in "Making Yourself Unforgettable" for Blacked'],
 	['cum-in-mouth', 1, 'Sarah Vandella in "Blow Bang Vandella" for HardX'],
 	['cum-on-butt', 0, 'Jynx Maze in "Don\'t Make Me Beg 4" for Evil Angel'],
+	['cum-on-boobs', 1, 'Kylie Page in "Melt In Your Mouth" for Twistys Hard'],
 	['da-tp', 5, 'Venera Maxima in LegalPorno GIO1287'],
 	['deepthroat', 2, 'Sarah Vandella for Throated'],
 	['dap', 7, 'Adriana Chechik in "DP Masters 6" for Jules Jordan'],
 	['double-blowjob', 1, 'Veronica Rodriguez and Penny Pax in "Fucking Older Guys 5" for Penthouse'],
 	['double-dildo', 0, 'Kali Roses in "Double Dildo Party" for KaliRoses.com'],
+	['double-dildo-anal', 0, 'Vina Sky and Kenzie Reeves in "Vina Sky\'s 1st Lesbian Anal" for HardX'],
 	['double-dildo-blowjob', 0, 'Adriana Chechik and Vicki Chase in "Anal Savages 1" for Jules Jordan'],
 	['dp', 3, 'Hime Marie in LegalPorno AA047'],
 	['dvp', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
@@ -626,13 +628,13 @@ const tagPosters = [
 	['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
 	['indian', 0, 'Resha in "Casting Resha" for Watch 4 Beauty'],
 	['interracial', 0, 'Jaye Summers and Prince Yahshua in "Platinum Pussy 3" for Jules Jordan'],
-	['latina', 2, 'Veronica Leal for Her Limit'],
+	['latina', 3, 'Gina Valentina and Veronica Rodriguez in "Squirting Latinas" for LesbianX'],
 	['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
 	['maid', 0, 'Whitney Wright in "Dredd Up Your Ass 2" for Jules Jordan'],
 	['milf', 1, 'Francesca Le for Evil Angel'],
 	['mff', 1, 'Anikka Albrite, Kelsi Monroe and Mick Blue for HardX'],
 	['mfm', 0, 'Vina Sky in "Jules Jordan\'s Three Ways" for Jules Jordan'],
-	['natural-boobs', 1, 'Nia Nacci for First Class POV'],
+	['natural-boobs', 3, 'Violet Starr in "Violet Starr 1st Lesbian Anal" for LesbianX'],
 	['nurse', 0, 'Sarah Vandella in "Cum For Nurse Sarah" for Brazzers'],
 	['oil', 2, 'Jade Kush for Passion HD'],
 	['oral-creampie', 1, 'Valentina Nappi for Her Limit'],
@@ -640,13 +642,16 @@ const tagPosters = [
 	['parody', 0, 'Capri Cavanni and Dani Daniels in "The Whore of Wall Street" for Brazzers'],
 	['piercings', 0, 'Kaegune in "When The Sun Goes Down" for Suicide Girls'],
 	['piss-drinking', 0, 'Scarlet Domingo in LegalPorno GL227'],
-	['pussy-eating', 1, 'Anikka Albrite and Riley Reid for In The Crack'],
+	['pussy-eating', 2, 'Anikka Albrite and Mia Malkova in "Big Anal Bombshells" for LesbianX'],
 	['redhead', 1, 'Lacy Lennon for Wicked'],
 	['squirting', 0, 'Veronica Rodriguez in "Hot Latina Squirting" for Jules Jordan'],
 	['schoolgirl', 1, 'Eliza Ibarra for Brazzers'],
 	['swallowing', 'poster'],
-	['teen', 0, 'Alexa Flexy for Sensual Girl'],
 	['tattoos', 1, 'Joanna Angel for Joanna Angel'],
+	['teen', 0, 'Alexa Flexy for Sensual Girl'],
+	['titty-fucking', 0, 'Kylie Page in "Stepsis Gives Soapy Handjob In Shower" for Spy Fam'],
+	['toy-anal', 1, 'Nina North and Cassidy Klein in "Nina\'s First Lesbian Anal" for LesbianX'],
+	['toy-dp', 1, 'Krissy Lynn and London River in "Lesbian DP Workout" for LesbianX'],
 	['trainbang', 'poster', 'Kali Roses in "Passing Me Around" for Blacked'],
 	['tap', 'poster', 'Kristy Black in SZ1986 for LegalPorno'],
 ]
@@ -662,7 +667,9 @@ const tagPosters = [
 	}));
 
 const tagPhotos = [
-	['69', 0, 'Abby Lee Brazil and Ramon Nomar for Wicked'],
+	['69', 3, 'Anne Amari and Alina Lopez in "Hot Lesbian Seduction" for LesbianX'],
+	['69', 1, 'Melody Petite for 21Sextury'],
+	['69', 2, 'Abigail Mac and Kissa Sins in "Lesbian Anal Workout" for HardX'],
 	['airtight', 5, 'Chloe Amour in "DP Masters 4" for Jules Jordan'],
 	['airtight', 1, 'Jynx Maze in "Pump My Ass Full of Cum 3" for Jules Jordan'],
 	['airtight', 2, 'Dakota Skye in "Dakota Goes Nuts" for ArchAngel'],
@@ -677,12 +684,18 @@ const tagPhotos = [
 	['anal', 3, 'Dakota Skye for Brazzers'],
 	// ['anal', 1, 'Veronica Leal and Tina Kay in "Agents On Anal Mission" for Asshole Fever'],
 	// ['anal', 0, 'Veronica Leal'],
+	['toy-anal', 0, 'Kira Noir in 1225 for InTheCrack'],
+	['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'],
 	['bts', 1, 'Madison Ivy in "Day With A Pornstar" for Brazzers'],
+	['blonde', 3, 'Kylie Page in "A Juicy Afternoon Delight" for New Sensations'],
 	['blonde', 2, 'Isabelle Deltore for Her Limit'],
 	['blowbang', 'poster', 'Marsha May in "Feeding Frenzy 12" for Jules Jordan'],
+	['blowjob', 1, 'Kylie Page in "Stepsis Gives Soapy Handjob In Shower" for Spy Fam'],
 	// ['bukkake', 'poster', 'Mia Malkova in "Facialized 2" for HardX'],
 	['caucasian', 0, 'Remy Lacroix for HardX'],
 	['caucasian', 1, 'Sheena Shaw for Brazzers'],
+	['cum-in-mouth', 0, 'Vina Sky and Avi Love for HardX'],
+	['cum-on-boobs', 0, 'Alessandra Jane for Private'],
 	['da-tp', 6, 'Adriana Chechik in "Gangbang Me" for HardX'],
 	['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
 	['da-tp', 1, 'Francys Belle in SZ1702 for LegalPorno'],
@@ -697,6 +710,8 @@ const tagPhotos = [
 	['deepthroat', 1, 'Jynx Maze in "Slutty and Sluttier 13" for Evil Angel'],
 	['deepthroat', 0, 'Chanel Grey in "Deepthroating Is Fun" for Throated'],
 	['double-blowjob', 0, 'Kira Noir and Kali Roses for Brazzers'],
+	['double-dildo-anal', 2, 'Adria Rae and Megan Rain in "Best Friends Anal" for Holed'],
+	['double-dildo-anal', 1, 'Sammie Rhodes and Ainsley Addision in "Tickle Me Pink" for We Live Together (Reality Kings)'],
 	['double-dildo-blowjob', 1, 'Aidra Fox and Reena Sky in "Reena\'s Got A Staring Problem" for Brazzers'],
 	['double-dildo-dp', 0, 'u/LacyCrow "Sometimes you have to do it yourself"'],
 	['dp', 2, 'Megan Rain in "DP Masters 4" for Jules Jordan'],
@@ -713,39 +728,48 @@ const tagPhotos = [
 	['facefucking', 4, 'Brooklyn Gray in "Throats Fucks 6" for Evil Angel'],
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
 	['fake-boobs', 10, 'Tia Cyrus in "Titty-Fucked Yoga Goddess" for Latina Sex Tapes'],
+	['fake-boobs', 11, 'Jessa Rhodes and Cali Carter in "Busty Anal Workout" for LesbianX'],
 	['fake-boobs', 9, 'Putri Cinta for StasyQ'],
-	['fake-boobs', 8, 'Amber Alena for Score'],
-	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
-	// ['fake-boobs', 6, 'Cathy Heaven in "Heavenly Ass" for Big Wett Butts'],
-	['fake-boobs', 7, 'Madison Ivy for Passion HD'],
-	['fake-boobs', 4, 'Capri Cavanni for Big Tits in Sports'],
 	['fake-boobs', 3, 'Ashly Anderson for Passion HD'],
+	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
+	['fake-boobs', 8, 'Amber Alena for Score'],
+	// ['fake-boobs', 6, 'Cathy Heaven in "Heavenly Ass" for Big Wett Butts'],
+	['fake-boobs', 4, 'Capri Cavanni for Big Tits in Sports'],
+	['fake-boobs', 7, 'Madison Ivy for Passion HD'],
 	['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
 	['gangbang', 0, '"4 On 1 Gangbangs" for Doghouse Digital'],
+	['gangbang', 6, 'Silvia Soprano in GIO1580 for LegalPorno'],
 	['gangbang', 4, 'Marley Brinx in "The Gangbang of Marley Brinx" for Jules Jordan'],
 	['gangbang', 1, 'Ginger Lynn in "Gangbang Mystique", a photoset shot by Suze Randall for Puritan No. 10, 1984. This photo pushed the boundaries of pornography at the time, as depicting a woman \'fully occupied\' was unheard of.'],
 	['gaping', 'poster', 'Zoey Monroe in "Manuel DPs Them All 5" for Jules Jordan'],
 	['gaping', 3, 'Jessyka Swan for 21Sextury'],
 	['gaping', 2, 'Alex Grey in "DP Masters 5" for Jules Jordan'],
+	['latina', 2, 'Veronica Leal for Her Limit'],
 	['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
 	['latina', 0, 'Vienna Black for Spizoo'],
+	['lesbian', 1, 'Kylie Page and Hadley Viscara in "Busty Blonde Bombshells" for LesbianX'],
+	['maid', 1, 'Alessandra Jane for Brazzers'],
 	// ['milf', 0, 'Olivia Austin in "Dredd 3" for Jules Jordan'],
 	['mff', 0, 'Madison Ivy, Adriana Chechik and Keiran Lee in "Day With A Pornstar" for Brazzers'],
 	['mfm', 6, 'Honey Gold in "Slut Puppies 12" for Jules Jordan'],
+	['natural-boobs', 1, 'Nia Nacci for First Class POV'],
 	['natural-boobs', 0, 'Valentina Nappi in "Hypnotic Curves" for LesbianX'],
+	['natural-boobs', 2, 'Kylie Page for All Girl Massage'],
 	['oil', 1, 'Kissa Sins in "Oil Overload 14" for JulesJordan'],
 	['oil', 3, 'Vina Sky for Lubed'],
 	['oil', 0, 'Jada Stevens in "Jada Stevens Anal Ass Oiled Up For James Deen\'s Cock" for Jules Jordan'],
 	['oral-creampie', 0, 'Henessy in "B(ass)t Friends" for Asshole Fever'],
 	['orgy', 0, 'Vicky Sol and Jolee Love in LegalPorno GIO1550'],
 	['orgy', 'poster', 'Zoey Mornoe (DP), Jillian Janson (sex), Frida Sante, Katerina Kay and  Natasha Starr in "Orgy Masters 6" for Jules Jordan'],
+	['pussy-eating', 3, 'Kylie Page and Kalina Ryu in "Training My Masseuse" for All Girl Massage'],
 	['pussy-eating', 0, 'Kali Roses licking Emily Willis\' pussy in "Peeping On My Neighbor" for Girl Girl'],
+	['pussy-eating', 1, 'Anikka Albrite and Riley Reid for In The Crack'],
 	['redhead', 0, 'Penny Pax in "The Submission of Emma Marx: Boundaries" for New Sensations'],
 	['tattoos', 0, 'Tigerlilly in "Wrapped In Blue" for Suicide Girls'],
 	['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
 	['tap', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
 	['tap', 2, 'Kira Thorn in GIO1018 for LegalPorno'],
-	['cum-in-mouth', 0, 'Vina Sky and Avi Love for HardX'],
+	['toy-dp', 0, 'Marley Brinx, Ivy Lebelle and Lyra Law in "Marley Brinx First GGDP" for LesbianX'],
 ]
 	.map(([slug, fileIndex, comment], index) => ({
 		id: `${slug}-${fileIndex}`,

src/media.js

@@ -200,6 +200,8 @@ async function findSourceDuplicates(baseMedias) {
 		knex('media').whereIn('source_page', extractUrls),
 	]);
 
+	console.log(sourceUrls, existingSourceMedia);
+
 	const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
 	const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
@@ -279,6 +281,7 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 }
 
 async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) {
+	try {
 		const thumbdir = path.join(media.role, 'thumbs', hashDir, hashSubDir);
 		const thumbpath = path.join(thumbdir, filename);
 
@@ -359,6 +362,13 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
 				height: info.height,
 			},
 		};
+	} catch (error) {
+		logger.error(`Failed to store ${media.id} from ${media.src} at ${filepath}: ${error.message}`);
+
+		await fsPromises.unlink(media.file.path);
+
+		return null;
+	}
 }
 
 async function storeFile(media) {
@@ -374,6 +384,15 @@ async function storeFile(media) {
 		const filedir = path.join(media.role, hashDir, hashSubDir);
 		const filepath = path.join(filedir, filename);
 
+		if (argv.force) {
+			try {
+				// remove old file to in case rename() does not overwrite (possibly on NFS setups)
+				await fsPromises.unlink(path.join(config.media.path, filepath));
+			} catch (error) {
+				// file probably didn't exist
+			}
+		}
+
 		if (media.meta.type === 'image') {
 			return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath);
 		}
@@ -383,6 +402,7 @@ async function storeFile(media) {
 			fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }),
 		]);
 
+		// move temp file to permanent location
 		await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
 
 		logger.silly(`Stored permanent media file for ${media.id} from ${media.src} at ${filepath}`);

src/updates.js

@@ -13,6 +13,16 @@ const { fetchIncludedEntities } = require('./entities');
 
 const emptyReleases = { uniqueReleases: [], duplicateReleases: [] };
 
+function mapReleasesToSiteIdAndEntryId(acc, release) {
+	const entityId = release.entityId || release.entity.id;
+	const entryId = release.entryId || release.entryId;
+
+	if (!acc[entityId]) acc[entityId] = {};
+	acc[entityId][entryId] = true;
+
+	return acc;
+}
+
 async function filterUniqueReleases(latestReleases, accReleases) {
 	const latestReleaseIdentifiers = latestReleases
 		.map(release => [release.entity.id, release.entryId]);
@@ -26,36 +36,37 @@ async function filterUniqueReleases(latestReleases, accReleases) {
 
 	// add entry IDs of accumulated releases to prevent an infinite scrape loop
 	// when one page contains the same release as the previous
-	const duplicateReleasesSiteIdAndEntryIds = duplicateReleases
+	const duplicateReleasesBySiteIdAndEntryId = duplicateReleases
 		.concat(accReleases)
-		.reduce((acc, release) => {
-			const entityId = release.entityId || release.entity.id;
-			const entryId = release.entryId || release.entryId;
+		.reduce(mapReleasesToSiteIdAndEntryId, {});
 
-			if (!acc[entityId]) acc[entityId] = {};
-			acc[entityId][entryId] = true;
+	const localDuplicateReleasesBySiteIdAndEntryId = accReleases.reduce(mapReleasesToSiteIdAndEntryId, {});
 
-			return acc;
-		}, {});
+	const uniqueReleases = latestReleases.filter(release => !duplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
+	const localUniqueReleases = latestReleases.filter(release => !localDuplicateReleasesBySiteIdAndEntryId[release.entity.id]?.[release.entryId]);
 
-	const uniqueReleases = latestReleases.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
-
-	return { uniqueReleases, duplicateReleases };
+	return {
+		uniqueReleases,
+		localUniqueReleases,
+		duplicateReleases,
+	};
 }
 
-function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, upcoming) {
+function needNextPage(releasesOnPage, uniqueReleasesOnPage, localUniqueReleasesOnPage, totalReleases, hasDates, upcoming) {
 	if (releasesOnPage.length === 0) {
 		return false;
 	}
 
 	if (upcoming) {
-		return uniqueReleases.length > 0 && argv.paginateUpcoming;
+		return uniqueReleasesOnPage.length > 0 && argv.paginateUpcoming;
 	}
 
+	// no longer works when there are no unique releases, need to keep track of /all/ releases regardless uniqueness
+	console.log(localUniqueReleasesOnPage.length);
+
+	if (localUniqueReleasesOnPage.length > 0) {
 		if (argv.last) {
-		// this will keep paginating until the second condition is met on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
-		// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
-		return releasesOnPage.length > 0 && totalReleases + releasesOnPage.length < argv.last;
+			return totalReleases + releasesOnPage.length < argv.last;
 		}
 
 		if (!hasDates) {
@@ -74,6 +85,7 @@ function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, u
 				return true;
 			}
 		}
+	}
 
 	return false;
 }
@@ -100,8 +112,8 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
 		|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
 		|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
 
-	const { uniqueReleases, duplicateReleases } = argv.force
-		? { uniqueReleases: limitedReleases, duplicateReleases: [] }
+	const { uniqueReleases, localUniqueReleases, duplicateReleases } = argv.force
+		? { uniqueReleases: limitedReleases, localUniqueReleases: limitedReleases, duplicateReleases: [] }
 		: await filterUniqueReleases(limitedReleases, acc.uniqueReleases);
 
 	const accReleases = {
@@ -109,7 +121,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
 		duplicateReleases: acc.duplicateReleases.concat(duplicateReleases),
 	};
 
-	if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) {
+	if (needNextPage(releases, uniqueReleases, localUniqueReleases, totalReleases, hasDates, upcoming)) {
 		return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases, totalReleases + releases.length);
 	}