boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added studio support. Added studios for LegalPorno. Improved media fetch error handling. Fixed DDFNetwork scraper, added media support.

This commit is contained in:
ThePendulum 2019-10-30 04:45:42 +01:00
parent d1ef444d75
commit 382e40b651
12 changed files with 254 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
assets/components/home/home.vue
View File

@ -91,14 +91,14 @@
>
<img
v-if="release.poster"
:src="`/${release.poster.path}`"
:src="`/media/${release.poster.path}`"
:alt="release.title"
class="scene-thumbnail"
>
<img
v-else-if="release.photos.length > 0"
:src="`/${release.photos[0].path}`"
:src="`/media/${release.photos[0].path}`"
:alt="release.title"
class="scene-thumbnail"
>

39
assets/components/release/release.vue
View File

@ -7,8 +7,8 @@
<div class="banner-trailer">
<video
v-if="release.trailer"
:src="`/${release.trailer.path}`"
:poster="`/${release.poster && release.poster.path}`"
:src="`/media/${release.trailer.path}`"
:poster="`/media/${(release.poster && release.poster.path) || (release.photos.length && release.photos[Math.floor(Math.random() * release.photos.length)].path)}`"
:alt="release.title"
class="banner-item"
controls
@ -18,12 +18,12 @@
<a
v-for="photo in photos"
:key="`banner-${photo.index}`"
:href="`/${photo.path}`"
:href="`/media/${photo.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/${photo.path}`"
:src="`/media/${photo.path}`"
:alt="`Photo ${photo.index + 1}`"
class="banner-item"
>
@ -64,22 +64,35 @@
<span class="row site">
<Icon icon="clapboard-play" />
<template v-if="release.studio">
<a
v-if="release.studio"
:href="release.studio.url"
target="_blank"
rel="noopener noreferrer"
class="site-link"
>{{ release.studio.name }}</a>,
</template>
<a
:href="release.network.url"
target="_blank"
rel="noopener noreferrer"
class="network-link"
>{{ release.network.name }}</a>:
<a
:href="release.site.url"
target="_blank"
rel="noopener noreferrer"
class="site-link"
>{{ release.site.name }}</a>
(<a
:href="release.network.url"
target="_blank"
rel="noopener noreferrer"
class="network-link"
>{{ release.network.name }}</a>)
</span>
<p class="row description">
<p
v-if="release.description"
class="row description"
>
<Icon icon="info2" />
{{ release.description }}
</p>
@ -89,7 +102,7 @@
<li
v-for="tag in release.tags"
:key="`tag-${tag.id}`"
:key="`tag-${tag.slug}`"
class="tag"
>
<a

22
migrations/20190325001339_releases.js
View File

@ -69,12 +69,26 @@ exports.up = knex => Promise.resolve()
table.string('name');
table.string('url');
table.text('description');
table.enum('orientation', ['gay', 'bi', 'lesbian', 'trans']);
table.string('parameters');
table.string('slug', 32)
.unique();
}))
.then(() => knex.schema.createTable('studios', (table) => {
table.increments('id', 12);
table.integer('network_id', 12)
.notNullable()
.references('id')
.inTable('networks');
table.string('name');
table.string('url');
table.text('description');
table.string('slug', 32)
.unique();
}))
.then(() => knex.schema.createTable('releases', (table) => {
table.increments('id', 16);
@ -83,6 +97,10 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('sites');
table.integer('studio_id', 12)
.references('id')
.inTable('studios');
table.string('shoot_id');
table.string('entry_id');
table.unique(['site_id', 'shoot_id']);
@ -109,7 +127,6 @@ exports.up = knex => Promise.resolve()
table.integer('rating')
.unsigned();
table.enum('orientation', ['gay', 'bi', 'lesbian', 'trans']);
table.boolean('deep');
table.datetime('created_at')
@ -179,5 +196,6 @@ exports.down = knex => Promise.resolve()
.then(() => knex.schema.dropTable('actors'))
.then(() => knex.schema.dropTable('releases'))
.then(() => knex.schema.dropTable('sites'))
.then(() => knex.schema.dropTable('studios'))
.then(() => knex.schema.dropTable('directors'))
.then(() => knex.schema.dropTable('networks'));

267
package-lock.json generated
View File

@ -1498,6 +1498,7 @@
"version": "6.9.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz",
"integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==",
"dev": true,
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
@ -1566,7 +1567,8 @@
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
"dev": true
},
"arch": {
"version": "2.1.1",
@ -1577,6 +1579,7 @@
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
"integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
"dev": true,
"requires": {
"delegates": "^1.0.0",
"readable-stream": "^2.0.6"
@ -1656,6 +1659,7 @@
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
"dev": true,
"requires": {
"safer-buffer": "~2.1.0"
}
@ -1701,7 +1705,8 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"dev": true
},
"assign-symbols": {
"version": "1.0.0",
@ -1735,7 +1740,8 @@
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
"dev": true
},
"atob": {
"version": "2.1.2",
@ -1759,12 +1765,14 @@
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
"dev": true
},
"aws4": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==",
"dev": true
},
"axobject-query": {
"version": "2.0.2",
@ -2358,6 +2366,7 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"dev": true,
"requires": {
"tweetnacl": "^0.14.3"
}
@ -2770,7 +2779,8 @@
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
"dev": true
},
"chalk": {
"version": "2.4.2",
@ -2824,7 +2834,8 @@
"chownr": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
"integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
"integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==",
"dev": true
},
"chrome-trace-event": {
"version": "1.0.2",
@ -2989,6 +3000,7 @@
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"dev": true,
"requires": {
"delayed-stream": "~1.0.0"
}
@ -3085,7 +3097,8 @@
"console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
"dev": true
},
"consolidate": {
"version": "0.15.1",
@ -3374,6 +3387,7 @@
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0"
}
@ -3413,11 +3427,6 @@
"resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
},
"deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
},
"deep-is": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
@ -3481,12 +3490,14 @@
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
},
"delegates": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
"dev": true
},
"depd": {
"version": "1.1.2",
@ -3522,11 +3533,6 @@
"repeating": "^2.0.0"
}
},
"detect-libc": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
},
"dev-null": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/dev-null/-/dev-null-0.1.1.tgz",
@ -3605,6 +3611,7 @@
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
"dev": true,
"requires": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
@ -4414,17 +4421,20 @@
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
"dev": true
},
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=",
"dev": true
},
"fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=",
"dev": true
},
"fast-levenshtein": {
"version": "2.0.6",
@ -4646,12 +4656,14 @@
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
"dev": true
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
@ -4731,14 +4743,6 @@
"universalify": "^0.1.0"
}
},
"fs-minipass": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz",
"integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==",
"requires": {
"minipass": "^2.2.1"
}
},
"fs-readdir-recursive": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz",
@ -5338,6 +5342,7 @@
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
"dev": true,
"requires": {
"aproba": "^1.0.3",
"console-control-strings": "^1.0.0",
@ -5352,12 +5357,14 @@
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
"dev": true
},
"is-fullwidth-code-point": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
"dev": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -5366,6 +5373,7 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
"dev": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -5376,6 +5384,7 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"dev": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -5419,6 +5428,7 @@
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0"
}
@ -5543,12 +5553,14 @@
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
"dev": true
},
"har-validator": {
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"dev": true,
"requires": {
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
@ -5594,7 +5606,8 @@
"has-unicode": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
"dev": true
},
"has-value": {
"version": "1.0.0",
@ -5732,6 +5745,7 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
@ -5785,14 +5799,6 @@
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
},
"ignore-walk": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz",
"integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==",
"requires": {
"minimatch": "^3.0.4"
}
},
"import-cwd": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/import-cwd/-/import-cwd-2.1.0.tgz",
@ -6246,7 +6252,8 @@
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
"dev": true
},
"is-unc-path": {
"version": "1.0.0",
@ -6290,7 +6297,8 @@
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
"dev": true
},
"js-base64": {
"version": "2.5.1",
@ -6333,7 +6341,8 @@
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
"dev": true
},
"jsesc": {
"version": "2.5.2",
@ -6349,12 +6358,14 @@
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
"dev": true
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
},
"json-stable-stringify-without-jsonify": {
"version": "1.0.1",
@ -6365,7 +6376,8 @@
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
"dev": true
},
"json5": {
"version": "2.1.0",
@ -6394,6 +6406,7 @@
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"dev": true,
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
@ -7039,30 +7052,6 @@
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
},
"minipass": {
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
"integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
},
"dependencies": {
"yallist": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
"integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A=="
}
}
},
"minizlib": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz",
"integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==",
"requires": {
"minipass": "^2.2.1"
}
},
"mississippi": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
@ -7205,16 +7194,6 @@
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
"dev": true
},
"needle": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz",
"integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==",
"requires": {
"debug": "^2.1.2",
"iconv-lite": "^0.4.4",
"sax": "^1.2.4"
}
},
"negotiator": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
@ -7328,23 +7307,6 @@
"resolved": "https://registry.npmjs.org/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz",
"integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA="
},
"node-pre-gyp": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
"integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
"requires": {
"detect-libc": "^1.0.2",
"mkdirp": "^0.5.1",
"needle": "^2.2.1",
"nopt": "^4.0.1",
"npm-packlist": "^1.1.6",
"npmlog": "^4.0.2",
"rc": "^1.2.7",
"rimraf": "^2.6.1",
"semver": "^5.3.0",
"tar": "^4"
}
},
"node-releases": {
"version": "1.1.22",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.22.tgz",
@ -7481,20 +7443,6 @@
"sort-keys": "^1.0.0"
}
},
"npm-bundled": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz",
"integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g=="
},
"npm-packlist": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz",
"integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==",
"requires": {
"ignore-walk": "^3.0.1",
"npm-bundled": "^1.0.1"
}
},
"npm-run-path": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
@ -7507,6 +7455,7 @@
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
"dev": true,
"requires": {
"are-we-there-yet": "~1.1.2",
"console-control-strings": "~1.1.0",
@ -7536,7 +7485,8 @@
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
"dev": true
},
"object-assign": {
"version": "4.1.1",
@ -8018,7 +7968,8 @@
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
"dev": true
},
"pg": {
"version": "7.9.0",
@ -8576,24 +8527,6 @@
"schema-utils": "^1.0.0"
}
},
"rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"requires": {
"deep-extend": "^0.6.0",
"ini": "~1.3.0",
"minimist": "^1.2.0",
"strip-json-comments": "~2.0.1"
},
"dependencies": {
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
}
}
},
"react": {
"version": "16.8.6",
"resolved": "https://registry.npmjs.org/react/-/react-16.8.6.tgz",
@ -8818,6 +8751,7 @@
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"dev": true,
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.8.0",
@ -8844,17 +8778,20 @@
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
"dev": true
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
"dev": true
},
"tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"dev": true,
"requires": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
@ -8863,7 +8800,8 @@
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"dev": true
}
}
},
@ -8947,6 +8885,7 @@
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
"integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
"dev": true,
"requires": {
"glob": "^7.1.3"
}
@ -9251,11 +9190,6 @@
}
}
},
"sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"scheduler": {
"version": "0.13.6",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.13.6.tgz",
@ -9744,27 +9678,11 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"sqlite3": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.0.6.tgz",
"integrity": "sha512-EqBXxHdKiwvNMRCgml86VTL5TK1i0IKiumnfxykX0gh6H6jaKijAXvE9O1N7+omfNSawR2fOmIyJZcfe8HYWpw==",
"requires": {
"nan": "~2.10.0",
"node-pre-gyp": "^0.11.0",
"request": "^2.87.0"
},
"dependencies": {
"nan": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
"integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
}
}
},
"sshpk": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
"integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
"dev": true,
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
@ -9945,7 +9863,8 @@
"strip-json-comments": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
"dev": true
},
"style-loader": {
"version": "0.23.1",
@ -10011,27 +9930,6 @@
"integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==",
"dev": true
},
"tar": {
"version": "4.4.8",
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
"integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
"requires": {
"chownr": "^1.1.1",
"fs-minipass": "^1.2.5",
"minipass": "^2.3.4",
"minizlib": "^1.1.1",
"mkdirp": "^0.5.0",
"safe-buffer": "^5.1.2",
"yallist": "^3.0.2"
},
"dependencies": {
"yallist": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
"integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A=="
}
}
},
"tarn": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/tarn/-/tarn-1.1.4.tgz",
@ -10373,6 +10271,7 @@
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"dev": true,
"requires": {
"safe-buffer": "^5.0.1"
}
@ -10380,7 +10279,8 @@
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
"dev": true
},
"type-check": {
"version": "0.3.2",
@ -10561,6 +10461,7 @@
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
"dev": true,
"requires": {
"punycode": "^2.1.0"
}
@ -10656,6 +10557,7 @@
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"dev": true,
"requires": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
@ -10987,6 +10889,7 @@
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
"dev": true,
"requires": {
"string-width": "^1.0.2 || 2"
}

1
package.json
View File

@ -85,7 +85,6 @@
"prop-types": "^15.7.2",
"react": "^16.8.6",
"react-dom": "^16.8.6",
"sqlite3": "^4.0.6",
"tough-cookie": "^3.0.1",
"tty-table": "^2.7.0",
"vue": "^2.6.10",

15
seeds/02_tags.js
View File

@ -54,6 +54,11 @@ exports.seed = knex => Promise.resolve()
const groupsMap = groups.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
return knex.raw(`${knex('tags').insert([
{
name: '4K',
slug: '4k',
alias_for: null,
},
{
name: '69',
slug: '69',
@ -808,6 +813,10 @@ exports.seed = knex => Promise.resolve()
name: 'anal',
alias_for: tagsMap['anal-sex'],
},
{
name: 'anal gape',
alias_for: tagsMap['gaping'],
},
{
name: 'anilingus',
alias_for: tagsMap['ass-licking'],
@ -816,6 +825,10 @@ exports.seed = knex => Promise.resolve()
name: 'asians',
alias_for: tagsMap['asian'],
},
{
name: 'ass fucking',
alias_for: tagsMap['anal-sex'],
},
{
name: 'atm',
alias_for: tagsMap['ass-to-mouth'],
@ -1125,7 +1138,7 @@ exports.seed = knex => Promise.resolve()
alias_for: tagsMap['feet'],
},
{
name: 'French kissing',
name: 'french kissing',
alias_for: tagsMap['kissing'],
},
{

18
src/fetch-releases.js
View File

@ -90,6 +90,8 @@ async function storeActors(release, releaseEntry) {
const actors = await knex('actors').whereIn('name', release.actors);
const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
console.log(release.actors, actors, newActors);
const { rows: insertedActors } = newActors.length
? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({
name: actorName,
@ -118,6 +120,8 @@ async function storePhotos(release, releaseEntry) {
const mimetype = mime.getType(pathname);
const res = await bhttp.get(photoUrl);
if (res.statusCode === 200) {
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
@ -125,11 +129,16 @@ async function storePhotos(release, releaseEntry) {
filepath,
mimetype,
};
}
console.warn(`Failed to store photo ${index + 1} for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
return null;
}, {
concurrency: 2,
});
await knex('media').insert(files.map(({ filepath, mimetype }, index) => ({
await knex('media').insert(files.filter(file => file).map(({ filepath, mimetype }, index) => ({
path: filepath,
mime: mimetype,
index,
@ -144,6 +153,7 @@ async function storePoster(release, releaseEntry) {
const res = await bhttp.get(release.poster);
if (res.statusCode === 200) {
const { pathname } = new URL(release.poster);
const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
@ -157,6 +167,11 @@ async function storePoster(release, releaseEntry) {
target_id: releaseEntry.id,
role: 'poster',
});
return;
}
console.warn(`Failed to store poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
}
async function storeTrailer(release, releaseEntry) {
@ -183,6 +198,7 @@ async function storeReleases(releases = []) {
return Promise.map(releases, async (release) => {
const curatedRelease = {
site_id: release.site.id,
studio_id: release.studio ? release.studio.id : null,
shoot_id: release.shootId || null,
entry_id: release.entryId || null,
url: release.url,

14
src/releases.js
View File

@ -41,6 +41,13 @@ async function curateRelease(release) {
name: release.site_name,
url: release.site_url,
},
studio: release.studio_id
? {
id: release.studio_id,
name: release.studio_name,
url: release.studio_url,
}
: null,
network: {
id: release.network_id,
name: release.network_name,
@ -58,8 +65,13 @@ async function fetchReleases(releaseId) {
const releases = await knex('releases')
.where(releaseId ? { 'releases.id': releaseId } : {})
.select('releases.*', 'sites.name as site_name', 'sites.url as site_url', 'sites.network_id', 'networks.name as network_name', 'networks.url as network_url')
.select(
'releases.*', 'sites.name as site_name', 'sites.url as site_url', 'sites.network_id',
'studios.name as studio_name', 'studios.url as studio_url',
'networks.name as network_name', 'networks.url as network_url',
)
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('studios', 'releases.studio_id', 'studios.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.orderBy('date', 'desc')
.limit(100);

53
src/scrapers/ddfnetwork.js
View File

@ -4,7 +4,7 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
// const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */
@ -18,10 +18,14 @@ function scrapeLatest(html, site) {
const url = `${site.url}${sceneLinkElement.attr('href')}`;
const entryId = url.split('/').slice(-1)[0];
const date = moment.utc($(element).find('.card-footer .text-muted').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray().filter(actor => actor);
const date = moment.utc($(element).find('small[datetime]').attr('datetime'), 'YYYY-MM-DD HH:mm:ss').toDate();
const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim())
.toArray()
.filter(actor => actor);
const duration = Number($(element).find('.card-info div:nth-child(2) .card-text').text().slice(0, -4)) * 60;
const duration = parseInt($(element).find('.card-info div:nth-child(2) .card-text').text(), 10) * 60;
const poster = sceneLinkElement.find('img').attr('data-src');
return {
url,
@ -30,6 +34,7 @@ function scrapeLatest(html, site) {
actors,
date,
duration,
poster,
rating: null,
site,
};
@ -40,26 +45,39 @@ async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const entryId = url.split('/').slice(-1)[0];
const title = $('.video-title h1').text();
const description = $('.description-box .box-container').text();
const title = $('meta[itemprop="name"]').attr('content');
const description = $('.descr-box p').text(); // meta tags don't contain full description
const date = moment.utc($('.video-title .remain time').text(), 'MMMM DD, YYYY').toDate();
const actors = $('.pornstars-box .pornstar-card .card-title a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const date = moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate();
const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($('.info-panel.likes .likes').text());
const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
const rawTags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const poster = $('#video').attr('poster');
const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
const trailer540 = $('source[res="540"]').attr('src');
const trailer720 = $('source[res="720"]').attr('src');
/*
* broken as of nov 2019
const { origin } = new URL($('.pornstar-card meta[itemprop="url"]').first().attr('content'));
const rawTags = $('#tagsBox .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [channelSite, tags] = await Promise.all([
// don't find site if original is already specific
site.isFallback ? knex('sites').where({ url: origin }).first() : site,
matchTags(rawTags),
]);
*/
const tags = await matchTags(rawTags);
return {
url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
// url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
url,
entryId,
title,
description,
@ -67,10 +85,23 @@ async function scrapeScene(html, url, site) {
date,
duration,
tags,
poster,
photos,
trailer: trailer540
? {
src: trailer540,
quality: 540,
}
: {
// backup
src: trailer720,
quality: 720,
},
rating: {
likes,
},
site: channelSite || site,
// site: channelSite || site,
site,
};
}

13
src/scrapers/legalporno.js
View File

@ -3,6 +3,7 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
@ -95,8 +96,17 @@ async function scrapeScene(html, url, site, useGallery) {
const trailer = data.clip.qualities.find(clip => clip.quality === 'vga' || clip.quality === 'hd');
const studioName = $('.watchpage-studioname').first().text().trim();
const studioSlug = studioName.replace(/\s+/g, '').toLowerCase();
const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = await matchTags(rawTags);
const [studio, tags] = await Promise.all([
knex('studios')
.where({ name: studioName })
.orWhere({ slug: studioSlug })
.first(),
matchTags(rawTags),
]);
return {
url,
@ -115,6 +125,7 @@ async function scrapeScene(html, url, site, useGallery) {
},
tags,
site,
studio,
};
}

6
src/tags.js
View File

@ -3,9 +3,13 @@
const knex = require('./knex');
async function matchTags(rawTags) {
const tags = rawTags
.concat(rawTags.map(tag => tag.toLowerCase()))
.concat(rawTags.map(tag => tag.toUpperCase()));
const tagEntries = await knex('tags')
.pluck('aliases.id')
.whereIn('tags.name', rawTags.map(tag => tag.toLowerCase()))
.whereIn('tags.name', tags)
.where(function where() {
this
.whereNull('tags.alias_for')

2
src/web/server.js
View File

@ -12,7 +12,7 @@ function initServer() {
const app = express();
const router = Router();
router.use(express.static(config.photoPath));
router.use('/media', express.static(config.photoPath));
router.use(express.static('public'));
router.use(bodyParser.json({ strict: false }));