DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Tweaked Spizoo scraper for Goth Girlfriends.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-09-13 01:22:46 +02:00
parent b41317706f
commit d3f15a6a2b
4 changed files with 106 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/sites.js
View File

@@ -71,7 +71,11 @@ async function findSiteByUrl(url) {
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
'networks.name as network_name',
'networks.slug as network_slug',
'networks.url as network_url',
'networks.description as network_description',
'networks.parameters as network_parameters',
)
.where('sites.url', url)
.orWhere('sites.url', origin)
@@ -114,7 +118,11 @@ async function fetchSitesFromArgv() {
const rawSites = await knex('sites')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
'networks.name as network_name',
'networks.slug as network_slug',
'networks.url as network_url',
'networks.description as network_description',
'networks.parameters as network_parameters',
)
.whereIn('sites.slug', argv.sites || [])
.orWhereIn('networks.slug', argv.networks || [])
@@ -133,7 +141,11 @@ async function fetchSitesFromConfig() {
const rawSites = await knex('sites')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
'networks.name as network_name',
'networks.slug as network_slug',
'networks.url as network_url',
'networks.description as network_description',
'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.where((builder) => {
@@ -168,7 +180,11 @@ async function fetchSites(queryObject) {
.where((builder) => whereOr(queryObject, 'sites', builder))
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
'networks.name as network_name',
'networks.slug as network_slug',
'networks.url as network_url',
'networks.description as network_description',
'networks.parameters as network_parameters',
)
.leftJoin('networks', 'sites.network_id', 'networks.id')
.limit(100);