DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

Browse Source
This commit is contained in:
ThePendulum
2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/utils/argv-include.js
View File

@@ -1,20 +1,20 @@
'use strict';
function include(argv) {
return {
covers: argv.media && argv.covers,
media: argv.media,
photos: argv.media && argv.photos,
poster: argv.media && argv.posters,
posters: argv.media && argv.posters,
releases: argv.withReleases,
scenes: argv.withReleases,
teaser: argv.media && argv.videos && argv.teasers,
teasers: argv.media && argv.videos && argv.teasers,
trailer: argv.media && argv.videos && argv.trailers,
trailers: argv.media && argv.videos && argv.trailers,
videos: argv.videos,
};
return {
covers: argv.media && argv.covers,
media: argv.media,
photos: argv.media && argv.photos,
poster: argv.media && argv.posters,
posters: argv.media && argv.posters,
releases: argv.withReleases,
scenes: argv.withReleases,
teaser: argv.media && argv.videos && argv.teasers,
teasers: argv.media && argv.videos && argv.teasers,
trailer: argv.media && argv.videos && argv.trailers,
trailers: argv.media && argv.videos && argv.trailers,
videos: argv.videos,
};
}
module.exports = include;

122
src/utils/buffer.js
View File

@@ -13,106 +13,106 @@ const file = 'https://speed.hetzner.de/100MB.bin';
// const file = 'https://speed.hetzner.de/10GB.bin';
function getMemoryUsage() {
return process.memoryUsage().rss / (10 ** 6);
return process.memoryUsage().rss / (10 ** 6);
}
const stats = {
peakMemoryUsage: getMemoryUsage(),
done: false,
downloads: {},
peakMemoryUsage: getMemoryUsage(),
done: false,
downloads: {},
};
function render() {
const downloads = Object.entries(stats.downloads);
const downloads = Object.entries(stats.downloads);
process.stdout.clearScreenDown();
process.stdout.clearScreenDown();
process.stdout.write(`peak memory: ${stats.peakMemoryUsage.toFixed(2)} MB\n`);
process.stdout.write(`peak memory: ${stats.peakMemoryUsage.toFixed(2)} MB\n`);
downloads.forEach(([download, progress]) => {
process.stdout.write(`${download}: ${progress}${typeof progress === 'string' ? '' : '%'}\n`);
});
downloads.forEach(([download, progress]) => {
process.stdout.write(`${download}: ${progress}${typeof progress === 'string' ? '' : '%'}\n`);
});
process.stdout.moveCursor(0, -(downloads.length + 1));
process.stdout.cursorTo(0);
process.stdout.moveCursor(0, -(downloads.length + 1));
process.stdout.cursorTo(0);
if (downloads.length === 0 || !downloads.every(([_label, download]) => typeof download === 'string')) {
setTimeout(() => render(), 1000);
return;
}
if (downloads.length === 0 || !downloads.every(([_label, download]) => typeof download === 'string')) {
setTimeout(() => render(), 1000);
return;
}
process.stdout.moveCursor(0, downloads.length + 1);
process.stdout.moveCursor(0, downloads.length + 1);
}
function setProgress(label, completedBytes, totalBytes, hash) {
const memory = getMemoryUsage();
const memory = getMemoryUsage();
stats.peakMemoryUsage = Math.max(memory, stats.peakMemoryUsage);
stats.downloads[label] = hash || Math.round((completedBytes / totalBytes) * 100);
stats.peakMemoryUsage = Math.max(memory, stats.peakMemoryUsage);
stats.downloads[label] = hash || Math.round((completedBytes / totalBytes) * 100);
}
async function buffered(label) {
const hash = new blake2.Hash('blake2b');
const hash = new blake2.Hash('blake2b');
const imageRes = await bhttp.get(file, {
onDownloadProgress(completedBytes, totalBytes) {
setProgress(label, completedBytes, totalBytes);
},
});
const imageRes = await bhttp.get(file, {
onDownloadProgress(completedBytes, totalBytes) {
setProgress(label, completedBytes, totalBytes);
},
});
hash.update(imageRes.body);
setProgress(label, null, null, hash.digest('hex'));
hash.update(imageRes.body);
setProgress(label, null, null, hash.digest('hex'));
await fsPromises.writeFile(`/mnt/stor/Pictures/traxxx/temp/buffered-${label}.bin`, imageRes.body);
await fsPromises.writeFile(`/mnt/stor/Pictures/traxxx/temp/buffered-${label}.bin`, imageRes.body);
}
async function streamed(label) {
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const hashStream = new PassThrough();
const targetStream = fs.createWriteStream(`/mnt/stor/Pictures/traxxx/temp/streamed-${label}.bin`);
const hashStream = new PassThrough();
const targetStream = fs.createWriteStream(`/mnt/stor/Pictures/traxxx/temp/streamed-${label}.bin`);
const imageRes = await bhttp.get(file, {
stream: true,
});
const imageRes = await bhttp.get(file, {
stream: true,
});
const stream = imageRes
.pipe(hashStream)
.pipe(targetStream);
const stream = imageRes
.pipe(hashStream)
.pipe(targetStream);
imageRes.on('progress', (completedBytes, totalBytes) => {
setProgress(label, completedBytes, totalBytes);
});
imageRes.on('progress', (completedBytes, totalBytes) => {
setProgress(label, completedBytes, totalBytes);
});
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
stream.on('finish', () => {
hash.end();
setProgress(label, null, null, hash.read());
});
stream.on('finish', () => {
hash.end();
setProgress(label, null, null, hash.read());
});
}
async function init() {
const n = argv.n || 1;
const n = argv.n || 1;
if (argv._.includes('stream')) {
console.log('using streams');
render();
if (argv._.includes('stream')) {
console.log('using streams');
render();
await Promise.map(Array.from({ length: n }), async (value, index) => streamed(index + 1));
await Promise.map(Array.from({ length: n }), async (value, index) => streamed(index + 1));
return;
}
return;
}
if (argv._.includes('buffer')) {
console.log('using buffers');
render();
if (argv._.includes('buffer')) {
console.log('using buffers');
render();
await Promise.map(Array.from({ length: n }), async (value, index) => buffered(index + 1));
}
await Promise.map(Array.from({ length: n }), async (value, index) => buffered(index + 1));
}
}
init();

16
src/utils/capitalize.js
View File

@@ -1,16 +1,16 @@
'use strict';
function capitalize(string, trim = true) {
if (!string) {
return '';
}
if (!string) {
return '';
}
const capitalized = string
.split(/\s+/)
.map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' ');
const capitalized = string
.split(/\s+/)
.map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' ');
return trim ? capitalized.trim() : capitalized;
return trim ? capitalized.trim() : capitalized;
}
module.exports = capitalize;

4
src/utils/chunk.js
View File

@@ -1,8 +1,8 @@
'use strict';
function chunk(array, chunkSize) {
return Array.from({ length: Math.ceil(array.length / chunkSize) })
.map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
return Array.from({ length: Math.ceil(array.length / chunkSize) })
.map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
}
module.exports = chunk;

42
src/utils/convert.js
View File

@@ -1,48 +1,48 @@
'use strict';
function inchesToCm(inches) {
return Math.round(Number(inches) * 2.54);
return Math.round(Number(inches) * 2.54);
}
function feetInchesToCm(feet, inches) {
if (typeof feet === 'string' && !inches) {
const [feetPart, inchesPart] = feet.match(/\d+/g);
return feetInchesToCm(feetPart, inchesPart);
}
if (typeof feet === 'string' && !inches) {
const [feetPart, inchesPart] = feet.match(/\d+/g);
return feetInchesToCm(feetPart, inchesPart);
}
return Math.round((Number(feet) * 30.48) + (Number(inches) * 2.54));
return Math.round((Number(feet) * 30.48) + (Number(inches) * 2.54));
}
function cmToFeetInches(centimeters) {
const feet = Math.floor(centimeters / 30.48);
const inches = Math.round((centimeters / 2.54) % (feet * 12));
const feet = Math.floor(centimeters / 30.48);
const inches = Math.round((centimeters / 2.54) % (feet * 12));
return { feet, inches };
return { feet, inches };
}
function heightToCm(height) {
const [feet, inches] = height.match(/\d+/g);
const [feet, inches] = height.match(/\d+/g);
return feetInchesToCm(feet, inches);
return feetInchesToCm(feet, inches);
}
function lbsToKg(lbs) {
const pounds = lbs.toString().match(/\d+/)[0];
const pounds = lbs.toString().match(/\d+/)[0];
return Math.round(Number(pounds) * 0.453592);
return Math.round(Number(pounds) * 0.453592);
}
function kgToLbs(kgs) {
const kilos = kgs.toString().match(/\d+/)[0];
const kilos = kgs.toString().match(/\d+/)[0];
return Math.round(Number(kilos) / 0.453592);
return Math.round(Number(kilos) / 0.453592);
}
module.exports = {
cmToFeetInches,
feetInchesToCm,
heightToCm,
inchesToCm,
lbsToKg,
kgToLbs,
cmToFeetInches,
feetInchesToCm,
heightToCm,
inchesToCm,
lbsToKg,
kgToLbs,
};

16
src/utils/cookies.js
View File

@@ -1,16 +1,16 @@
'use strict';
function cookieToData(cookieString) {
return cookieString.split('; ').reduce((acc, cookie) => {
const [key, value] = cookie.split('=');
return cookieString.split('; ').reduce((acc, cookie) => {
const [key, value] = cookie.split('=');
return {
...acc,
[key]: value,
};
}, {});
return {
...acc,
[key]: value,
};
}, {});
}
module.exports = {
cookieToData,
cookieToData,
};

12
src/utils/escape-html.js
View File

@@ -1,10 +1,10 @@
function escapeHtml(text) {
return text
.replace(/&/g, '&')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;');
return text
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;');
}
module.exports = escapeHtml;

136
src/utils/http.js
View File

@@ -11,107 +11,107 @@ const pipeline = util.promisify(stream.pipeline);
const logger = require('../logger')(__filename);
const defaultHeaders = {
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
};
const defaultOptions = {
responseTimeout: 30000,
responseTimeout: 30000,
};
const proxyAgent = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: config.proxy.port,
},
proxy: {
host: config.proxy.host,
port: config.proxy.port,
},
});
function useProxy(url) {
if (!config.proxy.enable) {
return false;
}
if (!config.proxy.enable) {
return false;
}
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
}
const queue = taskQueue();
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
logger.silly('Queueing requests');
});
queue.define('http', async ({
url,
method = 'GET',
body,
headers = {},
options = {},
url,
method = 'GET',
body,
headers = {},
options = {},
}) => {
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
}
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
}
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
};
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
};
if (useProxy(url)) {
reqOptions.agent = proxyAgent;
}
if (useProxy(url)) {
reqOptions.agent = proxyAgent;
}
const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
}
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
}
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
const json = Buffer.isBuffer(res.body) ? null : res.body;
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
const json = Buffer.isBuffer(res.body) ? null : res.body;
return {
...res,
originalRes: res,
html,
json,
pipe: res.pipe,
ok: res.statusCode >= 200 && res.statusCode <= 299,
code: res.statusCode,
status: res.statusCode,
};
return {
...res,
originalRes: res,
html,
json,
pipe: res.pipe,
ok: res.statusCode >= 200 && res.statusCode <= 299,
code: res.statusCode,
status: res.statusCode,
};
}, {
concurrency: 20,
concurrency: 20,
});
async function get(url, headers, options) {
return queue.push('http', {
method: 'GET',
url,
headers,
options,
});
return queue.push('http', {
method: 'GET',
url,
headers,
options,
});
}
async function post(url, body, headers, options) {
return queue.push('http', {
method: 'POST',
url,
body,
headers,
options,
});
return queue.push('http', {
method: 'POST',
url,
body,
headers,
options,
});
}
module.exports = {
get,
post,
get,
post,
};

8
src/utils/img.js
View File

@@ -7,12 +7,12 @@ const { argv } = require('yargs');
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
async function scan() {
console.log(url);
console.log(url);
const res = await bhttp.get(url);
const stats = await sharp(res.body).stats();
const res = await bhttp.get(url);
const stats = await sharp(res.body).stats();
console.log(stats);
console.log(stats);
}
scan();

38
src/utils/list.js
View File

@@ -4,33 +4,33 @@ const Promise = require('bluebird');
const knex = require('../knex');
async function listSites() {
const [networks, allSites] = await Promise.all([
knex('networks').orderBy('name'),
knex('sites').orderBy('name'),
]);
const [networks, allSites] = await Promise.all([
knex('networks').orderBy('name'),
knex('sites').orderBy('name'),
]);
await Promise.each(networks, async (network) => {
console.log(`* **${network.name}**`);
await Promise.each(networks, async (network) => {
console.log(`* **${network.name}**`);
const sites = await knex('sites')
.where({ network_id: network.id })
.orderBy('name');
const sites = await knex('sites')
.where({ network_id: network.id })
.orderBy('name');
if (sites.length === 1 && sites[0].name === network.name) {
return;
}
if (sites.length === 1 && sites[0].name === network.name) {
return;
}
sites.forEach((site) => {
const rkSpecial = network.id === 'realitykings'
sites.forEach((site) => {
const rkSpecial = network.id === 'realitykings'
&& (new URL(site.url).hostname === 'www.realitykings.com'
|| (site.parameters?.altLayout))
? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL
? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL
console.log(` * ${site.name}${rkSpecial}`);
});
});
console.log(` * ${site.name}${rkSpecial}`);
});
});
console.log(`${networks.length} networks with ${allSites.length} sites total`);
console.log(`${networks.length} networks with ${allSites.length} sites total`);
}
listSites();

132
src/utils/media.js
View File

@@ -12,99 +12,99 @@ const { PassThrough } = require('stream');
const http = require('./http');
function getMemoryUsage() {
return process.memoryUsage().rss / (10 ** 6);
return process.memoryUsage().rss / (10 ** 6);
}
let peakMemoryUsage = getMemoryUsage();
async function fetchSource(link) {
const id = nanoid();
const id = nanoid();
const hasher = new blake2.Hash('blake2b');
hasher.setEncoding('hex');
const hasher = new blake2.Hash('blake2b');
hasher.setEncoding('hex');
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough();
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
const tempFileStream = fs.createWriteStream(tempFilePath);
const hashStream = new PassThrough();
hashStream.on('data', chunk => hasher.write(chunk));
hashStream.on('data', chunk => hasher.write(chunk));
try {
const res = await http.get(link, null, {
stream: true,
transforms: [hashStream],
destination: tempFileStream,
timeout: 5000,
});
try {
const res = await http.get(link, null, {
stream: true,
transforms: [hashStream],
destination: tempFileStream,
timeout: 5000,
});
if (!res.ok) {
throw new Error(res.status);
}
if (!res.ok) {
throw new Error(res.status);
}
hasher.end();
const hash = hasher.read();
hasher.end();
const hash = hasher.read();
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
return {
id,
path: tempFilePath,
hash,
};
} catch (error) {
await fsPromises.unlink(tempFilePath);
return {
id,
path: tempFilePath,
hash,
};
} catch (error) {
await fsPromises.unlink(tempFilePath);
throw error;
}
throw error;
}
}
async function init() {
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
const links = linksFile.split('\n').filter(Boolean);
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
const links = linksFile.split('\n').filter(Boolean);
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
console.time('thumbs');
console.time('thumbs');
const files = await Promise.map(links, async (link) => {
try {
return await fetchSource(link);
} catch (error) {
console.log(`Failed to fetch ${link}: ${error.message}`);
return null;
}
});
const files = await Promise.map(links, async (link) => {
try {
return await fetchSource(link);
} catch (error) {
console.log(`Failed to fetch ${link}: ${error.message}`);
return null;
}
});
await Promise.map(files.filter(Boolean), async (file) => {
const image = sharp(file.path).jpeg();
await Promise.map(files.filter(Boolean), async (file) => {
const image = sharp(file.path).jpeg();
const [{ width, height }, { size }] = await Promise.all([
image.metadata(),
fsPromises.stat(file.path),
]);
const [{ width, height }, { size }] = await Promise.all([
image.metadata(),
fsPromises.stat(file.path),
]);
await Promise.all([
image
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
]);
await Promise.all([
image
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
image
.resize({
height: config.media.thumbnailSize,
withoutEnlargement: true,
})
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
]);
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
const memoryUsage = getMemoryUsage();
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
}, { concurrency: 10 });
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
}, { concurrency: 10 });
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
console.timeEnd('thumbs');
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
console.timeEnd('thumbs');
}
init();

16
src/utils/mofos.js
View File

@@ -6,16 +6,16 @@ const bhttp = require('bhttp');
const knex = require('../knex');
async function run() {
const network = await knex('networks').where('slug', 'mofos').first();
const sites = await knex('sites').where('network_id', network.id);
const network = await knex('networks').where('slug', 'mofos').first();
const sites = await knex('sites').where('network_id', network.id);
await Promise.map(sites, async (site) => {
const res = await bhttp.get(site.url);
await Promise.map(sites, async (site) => {
const res = await bhttp.get(site.url);
console.log(site.url, res.statusCode);
}, {
concurrency: 5,
});
console.log(site.url, res.statusCode);
}, {
concurrency: 5,
});
}
run();

2
src/utils/pick-random.js
View File

@@ -1,5 +1,5 @@
function pickRandom(array) {
return array[Math.floor(Math.random() * array.length)];
return array[Math.floor(Math.random() * array.length)];
}
module.exports = pickRandom;

42
src/utils/posters.js
View File

@@ -9,32 +9,32 @@ const argv = require('../argv');
const knex = require('../knex');
async function init() {
const posters = await knex('actors')
.select('actors.name as actor_name', 'releases.title', 'releases.date', 'media.path', 'media.index', 'sites.name as site_name', 'networks.name as network_name')
.whereIn('actors.name', (argv.actors || []).concat(argv._))
.join('releases_actors', 'releases_actors.actor_id', 'actors.id')
.join('releases', 'releases_actors.release_id', 'releases.id')
.join('sites', 'sites.id', 'releases.site_id')
.join('networks', 'networks.id', 'sites.network_id')
.join('releases_posters', 'releases_posters.release_id', 'releases.id')
.join('media', 'releases_posters.media_id', 'media.id');
// .join('releases_photos', 'releases_photos.release_id', 'releases.id')
// .join('media', 'releases_photos.media_id', 'media.id');
const posters = await knex('actors')
.select('actors.name as actor_name', 'releases.title', 'releases.date', 'media.path', 'media.index', 'sites.name as site_name', 'networks.name as network_name')
.whereIn('actors.name', (argv.actors || []).concat(argv._))
.join('releases_actors', 'releases_actors.actor_id', 'actors.id')
.join('releases', 'releases_actors.release_id', 'releases.id')
.join('sites', 'sites.id', 'releases.site_id')
.join('networks', 'networks.id', 'sites.network_id')
.join('releases_posters', 'releases_posters.release_id', 'releases.id')
.join('media', 'releases_posters.media_id', 'media.id');
// .join('releases_photos', 'releases_photos.release_id', 'releases.id')
// .join('media', 'releases_photos.media_id', 'media.id');
await Promise.all(posters.map(async (poster) => {
const source = path.join(config.media.path, poster.path);
await Promise.all(posters.map(async (poster) => {
const source = path.join(config.media.path, poster.path);
const directory = path.join(config.media.path, 'extracted', poster.actor_name);
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_')} (${moment.utc(poster.date).format('YYYY-MM-DD')})-${poster.index}.jpeg`);
await fs.mkdir(path.join(directory), { recursive: true });
const directory = path.join(config.media.path, 'extracted', poster.actor_name);
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_')} (${moment.utc(poster.date).format('YYYY-MM-DD')})-${poster.index}.jpeg`);
await fs.mkdir(path.join(directory), { recursive: true });
const file = await fs.readFile(source);
await fs.writeFile(target, file);
const file = await fs.readFile(source);
await fs.writeFile(target, file);
return file;
}));
return file;
}));
knex.destroy();
knex.destroy();
}
init();

424
src/utils/qu.js
View File

@@ -5,341 +5,341 @@ const moment = require('moment');
const http = require('./http');
function trim(str) {
if (!str) return null;
return str.trim().replace(/\s+/g, ' ');
if (!str) return null;
return str.trim().replace(/\s+/g, ' ');
}
function extractDate(dateString, format, match) {
if (match) {
const dateStamp = trim(dateString).match(match);
if (match) {
const dateStamp = trim(dateString).match(match);
if (dateStamp) {
const dateValue = moment.utc(dateStamp[0], format);
if (dateStamp) {
const dateValue = moment.utc(dateStamp[0], format);
return dateValue.isValid() ? dateValue.toDate() : null;
}
return null;
}
return dateValue.isValid() ? dateValue.toDate() : null;
}
return null;
}
const dateValue = moment.utc(trim(dateString), format);
const dateValue = moment.utc(trim(dateString), format);
return dateValue.isValid() ? dateValue.toDate() : null;
return dateValue.isValid() ? dateValue.toDate() : null;
}
function formatDate(dateValue, format, inputFormat) {
if (inputFormat) {
return moment(dateValue, inputFormat).format(format);
}
if (inputFormat) {
return moment(dateValue, inputFormat).format(format);
}
return moment(dateValue).format(format);
return moment(dateValue).format(format);
}
function prefixUrl(urlValue, origin, protocol = 'https') {
if (protocol && /^\/\//.test(urlValue)) {
return `${protocol}:${urlValue}`;
}
if (protocol && /^\/\//.test(urlValue)) {
return `${protocol}:${urlValue}`;
}
if (origin && /^\//.test(urlValue)) {
return `${origin}${urlValue}`;
}
if (origin && /^\//.test(urlValue)) {
return `${origin}${urlValue}`;
}
return urlValue;
return urlValue;
}
function q(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {
const value = selector
? context.querySelector(selector)?.[attr] || context.querySelector(selector)?.attributes[attr]?.value
: context[attr] || context.attributes[attr]?.value;
if (attr) {
const value = selector
? context.querySelector(selector)?.[attr] || context.querySelector(selector)?.attributes[attr]?.value
: context[attr] || context.attributes[attr]?.value;
return applyTrim && value ? trim(value) : value;
}
return applyTrim && value ? trim(value) : value;
}
return selector ? context.querySelector(selector) : context;
return selector ? context.querySelector(selector) : context;
}
function all(context, selector, attrArg, applyTrim = true) {
const attr = attrArg === true ? 'textContent' : attrArg;
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {
return Array.from(context.querySelectorAll(selector), el => q(el, null, attr, applyTrim));
}
if (attr) {
return Array.from(context.querySelectorAll(selector), el => q(el, null, attr, applyTrim));
}
return Array.from(context.querySelectorAll(selector));
return Array.from(context.querySelectorAll(selector));
}
function exists(context, selector) {
return !!q(context, selector);
return !!q(context, selector);
}
function html(context, selector) {
const el = q(context, selector, null, true);
const el = q(context, selector, null, true);
return el && el.innerHTML;
return el && el.innerHTML;
}
function texts(context, selector, applyTrim = true, filter = true) {
const el = q(context, selector, null, applyTrim);
if (!el) return null;
const el = q(context, selector, null, applyTrim);
if (!el) return null;
const nodes = Array.from(el.childNodes)
.filter(node => node.nodeName === '#text')
.map(node => (applyTrim ? trim(node.textContent) : node.textContent));
const nodes = Array.from(el.childNodes)
.filter(node => node.nodeName === '#text')
.map(node => (applyTrim ? trim(node.textContent) : node.textContent));
return filter ? nodes.filter(Boolean) : nodes;
return filter ? nodes.filter(Boolean) : nodes;
}
function text(context, selector, applyTrim = true) {
const nodes = texts(context, selector, applyTrim, true);
if (!nodes) return null;
const nodes = texts(context, selector, applyTrim, true);
if (!nodes) return null;
const textValue = nodes.join(' ');
const textValue = nodes.join(' ');
return applyTrim ? trim(textValue) : textValue;
return applyTrim ? trim(textValue) : textValue;
}
function meta(context, selector, attrArg = 'content', applyTrim = true) {
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, applyTrim);
}
if (/meta\[.*\]/.test(selector)) {
return q(context, selector, attrArg, applyTrim);
}
return q(context, `meta[${selector}]`, attrArg, applyTrim);
return q(context, `meta[${selector}]`, attrArg, applyTrim);
}
function date(context, selector, format, match, attr = 'textContent') {
const dateString = q(context, selector, attr, true);
const dateString = q(context, selector, attr, true);
if (!dateString) return null;
if (!dateString) return null;
return extractDate(dateString, format, match);
return extractDate(dateString, format, match);
}
function image(context, selector = 'img', attr = 'src', origin, protocol = 'https') {
const imageEl = q(context, selector, attr);
const imageEl = q(context, selector, attr);
// no attribute means q output will be HTML element
return attr ? prefixUrl(imageEl, origin, protocol) : imageEl;
// no attribute means q output will be HTML element
return attr ? prefixUrl(imageEl, origin, protocol) : imageEl;
}
function images(context, selector = 'img', attr = 'src', origin, protocol = 'https') {
const imageEls = all(context, selector, attr);
const imageEls = all(context, selector, attr);
return attr ? imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol)) : imageEls;
return attr ? imageEls.map(imageEl => prefixUrl(imageEl, origin, protocol)) : imageEls;
}
function url(context, selector = 'a', attr = 'href', origin, protocol = 'https') {
const urlEl = q(context, selector, attr);
const urlEl = q(context, selector, attr);
return attr ? prefixUrl(urlEl, origin, protocol) : urlEl;
return attr ? prefixUrl(urlEl, origin, protocol) : urlEl;
}
function urls(context, selector = 'a', attr = 'href', origin, protocol = 'https') {
const urlEls = all(context, selector, attr);
const urlEls = all(context, selector, attr);
return attr ? urlEls.map(urlEl => prefixUrl(urlEl, origin, protocol)) : urlEls;
return attr ? urlEls.map(urlEl => prefixUrl(urlEl, origin, protocol)) : urlEls;
}
function poster(context, selector = 'video', attr = 'poster', origin, protocol = 'https') {
const posterEl = q(context, selector, attr);
const posterEl = q(context, selector, attr);
return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
}
function video(context, selector = 'source', attr = 'src', origin, protocol = 'https') {
const trailerEl = q(context, selector, attr);
const trailerEl = q(context, selector, attr);
return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
}
function videos(context, selector = 'source', attr = 'src', origin, protocol = 'https') {
const trailerEls = all(context, selector, attr);
const trailerEls = all(context, selector, attr);
return attr ? trailerEls.map(trailerEl => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
return attr ? trailerEls.map(trailerEl => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
}
function duration(context, selector, match, attr = 'textContent') {
const durationString = q(context, selector, attr);
const durationString = q(context, selector, attr);
if (!durationString) return null;
const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
if (!durationString) return null;
const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
if (durationMatch) {
const segments = ['00'].concat(durationMatch[0].split(':')).slice(-3);
if (durationMatch) {
const segments = ['00'].concat(durationMatch[0].split(':')).slice(-3);
return moment.duration(segments.join(':')).asSeconds();
}
return moment.duration(segments.join(':')).asSeconds();
}
return null;
return null;
}
const legacyFuncs = {
q,
qa: all,
qall: all,
qd: date,
qdate: date,
qh: html,
qhtml: html,
qi: image,
qimage: image,
qimages: images,
qis: images,
ql: duration,
qlength: duration,
qm: meta,
qmeta: meta,
qp: poster,
qposter: poster,
qs: all,
qt: video,
qtext: text,
qtexts: texts,
qtrailer: video,
qtrailers: videos,
qts: videos,
qtx: text,
qtxs: texts,
qtxt: text,
qtxts: texts,
// qu: url,
qurl: url,
qurls: urls,
qus: urls,
q,
qa: all,
qall: all,
qd: date,
qdate: date,
qh: html,
qhtml: html,
qi: image,
qimage: image,
qimages: images,
qis: images,
ql: duration,
qlength: duration,
qm: meta,
qmeta: meta,
qp: poster,
qposter: poster,
qs: all,
qt: video,
qtext: text,
qtexts: texts,
qtrailer: video,
qtrailers: videos,
qts: videos,
qtx: text,
qtxs: texts,
qtxt: text,
qtxts: texts,
// qu: url,
qurl: url,
qurls: urls,
qus: urls,
};
const quFuncs = {
all,
html,
date,
dur: duration,
duration,
exists,
image,
images,
img: image,
imgs: images,
length: duration,
meta,
poster,
q,
text,
texts,
trailer: video,
url,
urls,
video,
videos,
all,
html,
date,
dur: duration,
duration,
exists,
image,
images,
img: image,
imgs: images,
length: duration,
meta,
poster,
q,
text,
texts,
trailer: video,
url,
urls,
video,
videos,
};
function init(element, window) {
if (!element) return null;
if (!element) return null;
const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (window && args[0] instanceof window.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (window && args[0] instanceof window.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (window && args[0] instanceof window.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
.reduce((acc, [key, func]) => ({
...acc,
[key]: (...args) => (window && args[0] instanceof window.HTMLElement // allow for different context
? func(...args)
: func(element, ...args)),
}), {});
return {
element,
el: element,
html: element.outerHTML || element.body.outerHTML,
text: trim(element.textContent),
...(window && {
window,
document: window.document,
}),
...legacyContextFuncs,
qu: quContextFuncs,
};
return {
element,
el: element,
html: element.outerHTML || element.body.outerHTML,
text: trim(element.textContent),
...(window && {
window,
document: window.document,
}),
...legacyContextFuncs,
qu: quContextFuncs,
};
}
function initAll(context, selector, window) {
if (Array.isArray(context)) {
return context.map(element => init(element, window));
}
if (Array.isArray(context)) {
return context.map(element => init(element, window));
}
return Array.from(context.querySelectorAll(selector))
.map(element => init(element, window));
return Array.from(context.querySelectorAll(selector))
.map(element => init(element, window));
}
function extract(htmlValue, selector) {
const { window } = new JSDOM(htmlValue);
const { window } = new JSDOM(htmlValue);
if (selector) {
return init(window.document.querySelector(selector), window);
}
if (selector) {
return init(window.document.querySelector(selector), window);
}
return init(window.document, window);
return init(window.document, window);
}
function extractAll(htmlValue, selector) {
const { window } = new JSDOM(htmlValue);
const { window } = new JSDOM(htmlValue);
return initAll(window.document, selector, window);
return initAll(window.document, selector, window);
}
async function get(urlValue, selector, headers, options, queryAll = false) {
const res = await http.get(urlValue, headers);
const res = await http.get(urlValue, headers);
if (res.statusCode === 200) {
const item = queryAll
? extractAll(res.body.toString(), selector)
: extract(res.body.toString(), selector);
if (res.statusCode === 200) {
const item = queryAll
? extractAll(res.body.toString(), selector)
: extract(res.body.toString(), selector);
return {
item,
items: all ? item : [item],
res,
ok: true,
status: res.statusCode,
};
}
return {
item,
items: all ? item : [item],
res,
ok: true,
status: res.statusCode,
};
}
return {
item: null,
items: [],
res,
ok: false,
status: res.statusCode,
};
return {
item: null,
items: [],
res,
ok: false,
status: res.statusCode,
};
}
async function getAll(urlValue, selector, headers, options) {
return get(urlValue, selector, headers, options, true);
return get(urlValue, selector, headers, options, true);
}
module.exports = {
extractDate,
extract,
extractAll,
init,
initAll,
formatDate,
get,
getAll,
context: init,
contextAll: initAll,
ed: extractDate,
ex: extract,
exa: extractAll,
fd: formatDate,
parseDate: extractDate,
ctx: init,
ctxa: initAll,
geta: getAll,
qu: quFuncs,
...legacyFuncs,
extractDate,
extract,
extractAll,
init,
initAll,
formatDate,
get,
getAll,
context: init,
contextAll: initAll,
ed: extractDate,
ex: extract,
exa: extractAll,
fd: formatDate,
parseDate: extractDate,
ctx: init,
ctxa: initAll,
geta: getAll,
qu: quFuncs,
...legacyFuncs,
};

29
src/utils/rename.js
View File

@@ -1,29 +0,0 @@
'use strict';
const path = require('path');
const Promise = require('bluebird');
const fs = require('fs-extra');
const fetchScene = require('../scrape-releases');
const argv = require('../argv');
async function renameFiles() {
const filenames = await fs.readdir(process.cwd());
const curated = await Promise.map(filenames, async (filename) => {
const shootId = filename.split(' ')[1];
const scene = await fetchScene(`https://kink.com/shoot/${shootId}`);
if (argv.confirm) {
await fs.rename(path.join(process.cwd(), filename), path.join(process.cwd(), `${scene.filename}.mp4`));
}
return scene.filename;
}, {
concurrency: 5,
});
console.log(curated);
}
renameFiles();

30
src/utils/resolve-place.js
View File

@@ -3,26 +3,26 @@
const bhttp = require('bhttp');
async function resolvePlace(query) {
if (!query) {
return null;
}
if (!query) {
return null;
}
const res = await bhttp.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`);
const [item] = res.body;
const res = await bhttp.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`);
const [item] = res.body;
if (item && item.address) {
const rawPlace = item.address;
const place = {};
if (item && item.address) {
const rawPlace = item.address;
const place = {};
if (rawPlace.city) place.city = rawPlace.city;
if (rawPlace.state) place.state = rawPlace.state;
if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase();
if (rawPlace.continent) place.continent = rawPlace.continent;
if (rawPlace.city) place.city = rawPlace.city;
if (rawPlace.state) place.state = rawPlace.state;
if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase();
if (rawPlace.continent) place.continent = rawPlace.continent;
return place;
}
return place;
}
return null;
return null;
}
module.exports = resolvePlace;

38
src/utils/scorelogos.js
View File

@@ -6,32 +6,32 @@ const fs = require('fs-extra');
const knex = require('../knex');
async function init() {
const sites = await knex('sites')
.select('networks.name', 'sites.slug')
.join('networks', 'networks.id', 'sites.network_id')
.where('networks.slug', 'score');
const sites = await knex('sites')
.select('networks.name', 'sites.slug')
.join('networks', 'networks.id', 'sites.network_id')
.where('networks.slug', 'score');
await Promise.map(sites, async (site) => {
const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
await Promise.map(sites, async (site) => {
const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
console.log(url);
console.log(url);
const res = await bhttp.get(url, {
responseTimeout: 5000,
});
const res = await bhttp.get(url, {
responseTimeout: 5000,
});
if (res.statusCode === 200) {
console.log(`Saving logo for ${site.slug}`);
if (res.statusCode === 200) {
console.log(`Saving logo for ${site.slug}`);
await fs.writeFile(`./score/${site.slug}.png`, res.body);
}
await fs.writeFile(`./score/${site.slug}.png`, res.body);
}
console.log(`No logo found for ${site.slug}`);
}, {
concurrency: 10,
});
console.log(`No logo found for ${site.slug}`);
}, {
concurrency: 10,
});
knex.destroy();
knex.destroy();
}
init();

12
src/utils/shuffle.js
View File

@@ -1,14 +1,14 @@
'use strict';
function shuffle(array) {
const shuffledArray = [...array];
const shuffledArray = [...array];
for (let i = array.length - 1; i > 0; i -= 1) {
const j = Math.floor(Math.random() * (i + 1));
[shuffledArray[i], shuffledArray[j]] = [shuffledArray[j], shuffledArray[i]];
}
for (let i = array.length - 1; i > 0; i -= 1) {
const j = Math.floor(Math.random() * (i + 1));
[shuffledArray[i], shuffledArray[j]] = [shuffledArray[j], shuffledArray[i]];
}
return shuffledArray;
return shuffledArray;
}
module.exports = shuffle;

34
src/utils/slugify.js
View File

@@ -1,30 +1,30 @@
'use strict';
function slugify(string, delimiter = '-', {
encode = false,
limit = 1000,
encode = false,
limit = 1000,
} = {}) {
if (!string) {
return string;
}
if (!string) {
return string;
}
const slugComponents = string.trim().toLowerCase().match(/\w+/g);
const slugComponents = string.trim().toLowerCase().match(/\w+/g);
if (!slugComponents) {
return '';
}
if (!slugComponents) {
return '';
}
const slug = slugComponents.reduce((acc, component, index) => {
const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`;
const slug = slugComponents.reduce((acc, component, index) => {
const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`;
if (accSlug.length < limit) {
return accSlug;
}
if (accSlug.length < limit) {
return accSlug;
}
return acc;
}, '');
return acc;
}, '');
return encode ? encodeURI(slug) : slug;
return encode ? encodeURI(slug) : slug;
}
module.exports = slugify;

58
src/utils/stream.js
View File

@@ -11,45 +11,45 @@ const sharp = require('sharp');
const url = 'https://thumbs.julesjordan.com/trial/content//upload/dl03/julesjordan/oil_overload_16_scene2//photos/alina_lopez_jules_jordan_com_77.jpg';
async function init() {
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const res = await bhttp.get(url, {
stream: true,
});
const res = await bhttp.get(url, {
stream: true,
});
const metaStream = sharp();
const hashStream = new PassThrough();
const target = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina.jpg'));
const thumbTarget = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina_thumb.jpg'));
const metaStream = sharp();
const hashStream = new PassThrough();
const target = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina.jpg'));
const thumbTarget = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina_thumb.jpg'));
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
metaStream.clone()
.resize(320)
.pipe(thumbTarget);
metaStream.clone()
.resize(320)
.pipe(thumbTarget);
const stream = res
.pipe(metaStream)
.pipe(hashStream)
.pipe(target);
const stream = res
.pipe(metaStream)
.pipe(hashStream)
.pipe(target);
stream.on('finish', () => {
hash.end();
const digest = hash.read();
stream.on('finish', () => {
hash.end();
const digest = hash.read();
console.log('stream', digest);
});
console.log('stream', digest);
});
metaStream.on('info', (info) => {
console.log('info', info);
});
metaStream.on('info', (info) => {
console.log('info', info);
});
const stats = await metaStream.stats();
const stats = await metaStream.stats();
console.log('stats', stats);
console.log('stats', stats);
}
init();

16
src/utils/timeout.js
View File

@@ -6,15 +6,15 @@ const sleep = 5000;
const timeout = 1000;
async function init() {
try {
const res = await bhttp.get(`https://httpstat.us/200?sleep=${sleep}`, {
responseTimeout: timeout,
});
try {
const res = await bhttp.get(`https://httpstat.us/200?sleep=${sleep}`, {
responseTimeout: timeout,
});
console.log(res.statusCode);
} catch (error) {
console.log(error);
}
console.log(res.statusCode);
} catch (error) {
console.log(error);
}
}
/*

20
src/utils/titles.js
View File

@@ -4,18 +4,18 @@ const argv = require('../argv');
const knex = require('../knex');
async function printTitles() {
const titles = await knex('releases')
.where((builder) => {
if (argv.sites) builder.whereIn('sites.slug', argv.sites);
if (argv.networks) builder.orWhereIn('networks.slug', argv.networks);
})
.join('sites', 'sites.id', 'releases.site_id')
.join('networks', 'networks.id', 'sites.network_id')
.pluck('title');
const titles = await knex('releases')
.where((builder) => {
if (argv.sites) builder.whereIn('sites.slug', argv.sites);
if (argv.networks) builder.orWhereIn('networks.slug', argv.networks);
})
.join('sites', 'sites.id', 'releases.site_id')
.join('networks', 'networks.id', 'sites.network_id')
.pluck('title');
console.log(titles.join('\n'));
console.log(titles.join('\n'));
knex.destroy();
knex.destroy();
}
printTitles();

76
src/utils/upsert.js
View File

@@ -4,54 +4,54 @@ const knex = require('../knex');
const logger = require('../logger')(__filename);
async function upsert(table, items, identifier = ['id'], _knex) {
const identifiers = Array.isArray(identifier) ? identifier : [identifier];
const identifiers = Array.isArray(identifier) ? identifier : [identifier];
const duplicates = await knex(table).whereIn(identifiers, items.map(item => identifiers.map(identifierX => item[identifierX])));
const duplicatesByIdentifiers = duplicates.reduce((acc, duplicate) => {
const duplicateIdentifier = identifiers.map(identifierX => duplicate[identifierX]).toString();
const duplicates = await knex(table).whereIn(identifiers, items.map(item => identifiers.map(identifierX => item[identifierX])));
const duplicatesByIdentifiers = duplicates.reduce((acc, duplicate) => {
const duplicateIdentifier = identifiers.map(identifierX => duplicate[identifierX]).toString();
return { ...acc, [duplicateIdentifier]: duplicate };
}, {});
return { ...acc, [duplicateIdentifier]: duplicate };
}, {});
const { insert, update } = items.reduce((acc, item) => {
const itemIdentifier = identifiers.map(identifierX => item[identifierX]).toString();
const { insert, update } = items.reduce((acc, item) => {
const itemIdentifier = identifiers.map(identifierX => item[identifierX]).toString();
if (duplicatesByIdentifiers[itemIdentifier]) {
acc.update.push(item);
return acc;
}
if (duplicatesByIdentifiers[itemIdentifier]) {
acc.update.push(item);
return acc;
}
acc.insert.push(item);
return acc;
}, {
insert: [],
update: [],
});
acc.insert.push(item);
return acc;
}, {
insert: [],
update: [],
});
if (knex) {
logger.debug(`${table}: Inserting ${insert.length}`);
logger.debug(`${table}: Updating ${update.length}`);
if (knex) {
logger.debug(`${table}: Inserting ${insert.length}`);
logger.debug(`${table}: Updating ${update.length}`);
const [inserted, updated] = await Promise.all([
knex(table).returning('*').insert(insert),
knex.transaction(async trx => Promise.all(update.map((item) => {
const clause = identifiers.reduce((acc, identifierX) => ({ ...acc, [identifierX]: item[identifierX] }), {});
const [inserted, updated] = await Promise.all([
knex(table).returning('*').insert(insert),
knex.transaction(async trx => Promise.all(update.map((item) => {
const clause = identifiers.reduce((acc, identifierX) => ({ ...acc, [identifierX]: item[identifierX] }), {});
return trx
.where(clause)
.update(item)
.into(table)
.returning('*');
}))),
]);
return trx
.where(clause)
.update(item)
.into(table)
.returning('*');
}))),
]);
return {
inserted: Array.isArray(inserted) ? inserted : [],
updated: updated.reduce((acc, updatedItems) => acc.concat(updatedItems), []),
};
}
return {
inserted: Array.isArray(inserted) ? inserted : [],
updated: updated.reduce((acc, updatedItems) => acc.concat(updatedItems), []),
};
}
return { insert, update };
return { insert, update };
}
module.exports = upsert;

30
src/utils/where-or.js
View File

@@ -1,25 +1,25 @@
'use strict';
function whereOr(query, table, builder) {
if (!query) {
return {};
}
if (!query) {
return {};
}
Object.entries(query).forEach(([key, value]) => {
if (value === undefined) {
return builder;
}
Object.entries(query).forEach(([key, value]) => {
if (value === undefined) {
return builder;
}
if (Array.isArray(value)) {
builder.orWhereIn(`${table}.${key}`, value);
return builder;
}
if (Array.isArray(value)) {
builder.orWhereIn(`${table}.${key}`, value);
return builder;
}
builder.orWhere(`${table}.${key}`, value);
return builder;
});
builder.orWhere(`${table}.${key}`, value);
return builder;
});
return builder;
return builder;
}
module.exports = whereOr;