Added Bang! scraper. Showing studio on tile. Added favicons to tiles.

This commit is contained in:
ThePendulum 2020-01-06 05:19:38 +01:00
parent 3a7aa58abb
commit bb5b8085f6
90 changed files with 430 additions and 45 deletions

View File

@ -68,8 +68,16 @@ import Sites from '../sites/sites.vue';
async function fetchNetwork() {
this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug);
if (this.network.studios) {
this.studios = this.network.studios.map(studio => ({
...studio,
network: this.network,
}));
}
this.sites = this.network.sites
.filter(site => !site.independent)
.concat(this.studios)
.sort(({ name: nameA }, { name: nameB }) => nameA.localeCompare(nameB));
this.releases = this.network.sites.map(site => site.releases).flat();
@ -89,7 +97,8 @@ export default {
data() {
return {
network: null,
sites: null,
sites: [],
studios: [],
releases: [],
pageTitle: null,
expanded: false,
@ -158,12 +167,13 @@ export default {
}
.header {
background: $profile;
width: 100%;
height: 4rem;
height: 3rem;
display: none;
flex-shrink: 0;
justify-content: center;
border-bottom: solid 1px $shadow-hint;
background: $profile;
}
.sites.compact {

View File

@ -29,10 +29,18 @@ export default {
</script>
<style lang="scss" scoped>
@import 'theme';
.networks {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem;
padding: 1rem;
}
@media(max-width: $breakpoint) {
.networks {
grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr));
}
}
</style>

View File

@ -80,7 +80,21 @@
<span class="chain">presents</span>
<a :href="`/site/${release.site.slug}`">
<a
v-if="release.studio"
:href="`/site/${release.studio.slug}`"
>
<img
:src="`/img/logos/${release.network.slug}/${release.studio.slug}.png`"
:title="release.studio.name"
class="logo logo-site"
>
</a>
<a
v-else
:href="`/site/${release.site.slug}`"
>
<img
:src="`/img/logos/${release.network.slug}/${release.site.slug}.png`"
:title="release.site.name"

View File

@ -11,12 +11,24 @@
class="site site-link"
>{{ release.network.name }}</router-link>
<router-link
v-else-if="release.studio"
:to="`/network/${release.network.slug}`"
class="site site-link"
><img
:src="`/img/logos/${release.network.slug}/favicon.png`"
class="favicon"
>{{ release.studio.name }}</router-link>
<router-link
v-else
v-tooltip.bottom="`Part of ${release.network.name}`"
:title="`Part of ${release.network.name}`"
:to="`/site/${release.site.slug}`"
class="site site-link"
><img
:src="`/img/logos/${release.network.slug}/favicon.png`"
class="favicon"
>{{ release.site.name }}</router-link>
<a
@ -178,11 +190,18 @@ export default {
justify-content: space-between;
position: absolute;
font-size: 0;
.favicon {
height: 1rem;
margin: 0 .25rem 0 0;
}
}
.site,
.date {
color: #fff;
display: flex;
align-items: center;
background: $shadow;
position: relative;
font-size: .8rem;

View File

@ -37,6 +37,10 @@ function curateSite(site, network) {
if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release));
if (site.network || network) curatedSite.network = site.network || network;
if (site.parameters) {
curatedSite.independent = !!JSON.parse(site.parameters).independent;
}
return curatedSite;
}
@ -52,6 +56,14 @@ function curateNetwork(network) {
curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
}
if (network.studios) {
curatedNetwork.studios = network.studios;
}
if (network.parameters) {
curatedNetwork.parameters = JSON.parse(network.parameters);
}
return curatedNetwork;
}

View File

@ -113,6 +113,12 @@ const releasesFragment = `
${releaseTagsFragment}
${releasePosterFragment}
${siteFragment}
studio {
id
name
slug
url
}
}
`;

View File

@ -22,6 +22,7 @@ function initNetworksActions(store, _router) {
slug
url
${releasesFragment}
parameters
network {
id
name
@ -29,6 +30,12 @@ function initNetworksActions(store, _router) {
url
}
}
studios {
id
name
slug
url
}
}
}
`, {

View File

@ -51,7 +51,7 @@ module.exports = {
'vixen',
'xempire',
],
fetchAfter: [3, 'months'],
fetchAfter: [1, 'week'],
columns: [
{
value: 'date',

51
package-lock.json generated
View File

@ -2626,6 +2626,14 @@
"toidentifier": "1.0.0"
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"mime-db": {
"version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
@ -4648,6 +4656,17 @@
"chardet": "^0.7.0",
"iconv-lite": "^0.4.24",
"tmp": "^0.0.33"
},
"dependencies": {
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"dev": true,
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
}
}
},
"extglob": {
@ -6217,9 +6236,9 @@
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz",
"integrity": "sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
@ -7746,6 +7765,14 @@
"ms": "^2.1.1"
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -9236,6 +9263,14 @@
"toidentifier": "1.0.0"
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"setprototypeof": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@ -12039,6 +12074,16 @@
"integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==",
"requires": {
"iconv-lite": "0.4.24"
},
"dependencies": {
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
}
}
},
"whatwg-mimetype": {

View File

@ -85,6 +85,7 @@
"face-api.js": "^0.21.0",
"fs-extra": "^7.0.1",
"graphile-utils": "^4.5.6",
"iconv-lite": "^0.5.0",
"jsdom": "^15.2.1",
"knex": "^0.16.5",
"knex-migrate": "^1.7.4",

View File

@ -149,9 +149,17 @@
position: absolute;
font-size: 0;
}
.details .favicon[data-v-3abcf101] {
height: 1rem;
margin: 0 .25rem 0 0;
}
.site[data-v-3abcf101],
.date[data-v-3abcf101] {
color: #fff;
display: -webkit-box;
display: flex;
-webkit-box-align: center;
align-items: center;
background: rgba(0, 0, 0, 0.5);
position: relative;
font-size: .8rem;
@ -684,13 +692,14 @@
filter: drop-shadow(1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(-1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 1px 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 -1px 0 rgba(255, 255, 255, 0.2));
}
.header[data-v-e2e12602] {
background: #222;
width: 100%;
height: 4rem;
height: 3rem;
display: none;
flex-shrink: 0;
-webkit-box-pack: center;
justify-content: center;
border-bottom: solid 1px rgba(0, 0, 0, 0.1);
background: #222;
}
.sites.compact[data-v-e2e12602] {
display: none;
@ -786,12 +795,18 @@
margin: 0;
}
/* $primary: #ff886c; */
.networks[data-v-4709d404] {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem;
padding: 1rem;
}
@media (max-width: 720px) {
.networks[data-v-4709d404] {
grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr));
}
}
/* $primary: #ff886c; */
.photos[data-v-0a0430c7] {

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 948 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 882 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 165 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 101 KiB

After

Width:  |  Height:  |  Size: 101 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 90 KiB

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.6 KiB

After

Width:  |  Height:  |  Size: 5.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 547 KiB

After

Width:  |  Height:  |  Size: 105 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.6 KiB

After

Width:  |  Height:  |  Size: 7.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 95 KiB

After

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 104 KiB

After

Width:  |  Height:  |  Size: 104 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 240 KiB

After

Width:  |  Height:  |  Size: 240 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 124 KiB

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.2 KiB

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 139 KiB

After

Width:  |  Height:  |  Size: 139 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 85 KiB

After

Width:  |  Height:  |  Size: 85 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

View File

@ -8,6 +8,11 @@ const networks = [
url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
},
{
slug: 'bang',
name: 'Bang!',
url: 'https://bang.com',
},
{
slug: 'bangbros',
name: 'Bang Bros',

View File

@ -119,6 +119,15 @@ function getSites(networksMap) {
name: 'Sweet Sophie Moone',
network_id: networksMap['21sextury'],
},
// BANG
{
slug: 'bang',
network_id: networksMap.bang,
name: 'Bang!',
url: 'https://bang.com',
description: null,
parameters: JSON.stringify({ independent: true }),
},
// BANGBROS
{
slug: 'assparade',
@ -638,7 +647,7 @@ function getSites(networksMap) {
},
{
slug: 'milfslikeitbig',
name: 'Milfs Like it Big',
name: 'MILFs Like It Big',
url: 'https://www.brazzers.com/sites/view/id/36/milfs-like-it-big',
description: "When hubby's away milfy will play. These bored housewives want to get fucked and they want it now. They're experienced and know what they want. America's suburbs are full of these cum-de-sacs just waiting to get laid. Their round tits and thick asses are just begging for it. Cum inside, but don't park out front!",
network_id: networksMap.brazzers,
@ -2279,6 +2288,14 @@ function getSites(networksMap) {
slug: 'girlsofnaked',
network_id: networksMap.realitykings,
},
{
name: 'Lil Humpers',
url: 'https://lilhumpers.com',
description: '',
parameters: JSON.stringify({ siteId: 310 }),
slug: 'lilhumpers',
network_id: networksMap.realitykings,
},
{
name: 'Mike in Brazil',
url: 'https://www.realitykings.com/scenes?site=24',

View File

@ -2,6 +2,91 @@ const upsert = require('../src/utils/upsert');
function getStudios(networksMap) {
return [
// Bang!
{
name: 'Trickery',
slug: 'bangtrickery',
url: 'https://www.bang.com/original/4800/bang-trickery',
network_id: networksMap.bang,
},
{
name: 'Yngr',
slug: 'yngrcom',
url: 'https://www.bang.com/original/5010/bang-yngr',
network_id: networksMap.bang,
},
{
name: 'Roadside XXX',
slug: 'bangroadsidexxx',
url: 'https://www.bang.com/original/4864/roadside-xxx',
network_id: networksMap.bang,
},
{
name: 'Surprise',
slug: 'bangsurprise',
url: 'https://www.bang.com/original/5000/bang-surprise',
network_id: networksMap.bang,
},
{
name: 'Real Teens',
slug: 'bangrealteens',
url: 'https://www.bang.com/original/3366/bang-real-teens',
network_id: networksMap.bang,
},
{
name: 'FCK.news',
slug: 'bangfcknews',
url: 'https://www.bang.com/original/4998/bang-fckNews',
network_id: networksMap.bang,
},
{
name: 'Pretty & Raw',
slug: 'prettyandraw',
url: 'https://www.bang.com/original/4792/bang-pretty-and-raw',
network_id: networksMap.bang,
},
{
name: 'Japan',
slug: 'bangjapan',
url: 'https://www.bang.com/original/3079/bang-japan',
network_id: networksMap.bang,
},
{
name: 'Rammed',
slug: 'bangrammed',
url: 'https://www.bang.com/original/4836/bang-rammed',
network_id: networksMap.bang,
},
{
name: 'Glamkore',
slug: 'bangglamkore',
url: 'https://www.bang.com/original/4586/bang-glamkore',
network_id: networksMap.bang,
},
{
name: 'Screw The Cops',
slug: 'bangscrewthecops',
url: 'https://www.bang.com/original/4710/bang-screw-cops',
network_id: networksMap.bang,
},
{
name: 'Real MILFs',
slug: 'bangrealmilfs',
url: 'https://www.bang.com/original/4448/bang-real-milfs',
network_id: networksMap.bang,
},
{
name: 'Confessions',
slug: 'bangconfessions',
url: 'https://www.bang.com/original/4308/bang-confessions',
network_id: networksMap.bang,
},
{
name: 'Casting',
slug: 'bangcasting',
url: 'https://www.bang.com/original/3261/bang-casting',
network_id: networksMap.bang,
},
// LegalPorno
{
slug: 'gonzocom',

View File

@ -291,6 +291,6 @@ exports.seed = knex => Promise.resolve()
return Promise.all([
upsert('tags_posters', tagPosterEntries, 'tag_id', knex),
upsert('tags_photos', tagPhotoEntries, 'tag_id', knex),
upsert('tags_photos', tagPhotoEntries, ['tag_id', 'media_id'], knex),
]);
});

View File

@ -1646,6 +1646,7 @@ const countries = [
},
{
name: 'United Kingdom of Great Britain and Northern Ireland',
alias: 'United Kingdom',
code: 826,
alpha2: 'GB',
alpha3: 'GBR',

View File

@ -180,6 +180,23 @@ async function attachChannelSite(release) {
};
}
async function attachStudio(release) {
if (!release.studio) {
return release;
}
const studio = await knex('studios')
.where('name', release.studio)
.orWhere('slug', release.studio)
.orWhere('url', release.studio)
.first();
return {
...release,
studio,
};
}
async function curateReleaseEntry(release) {
const curatedRelease = {
site_id: release.site.id,
@ -295,27 +312,31 @@ async function storeReleaseAssets(release, releaseId) {
await createMediaDirectory('releases', subpath);
try {
await Promise.all([
storePhotos(release.photos, {
targetId: releaseId,
subpath,
primaryRole: release.poster ? null : 'poster',
}, identifier),
release.poster && storePhotos([release.poster], {
// don't use Promise.all to prevent concurrency issues with duplicate detection
if (release.poster) {
await storePhotos([release.poster], {
role: 'poster',
targetId: releaseId,
subpath,
}, identifier),
storePhotos(release.covers, {
role: 'cover',
targetId: releaseId,
subpath,
}, identifier),
storeTrailer(release.trailer, {
targetId: releaseId,
subpath,
}, identifier),
]);
}, identifier);
}
await storePhotos(release.photos, {
targetId: releaseId,
subpath,
primaryRole: release.poster ? null : 'poster',
}, identifier);
await storePhotos(release.covers, {
role: 'cover',
targetId: releaseId,
subpath,
}, identifier);
await storeTrailer(release.trailer, {
targetId: releaseId,
subpath,
}, identifier);
} catch (error) {
console.log(release.url, error);
}
@ -363,7 +384,8 @@ async function storeReleases(releases) {
const storedReleases = await Promise.map(releases, async (release) => {
try {
const releaseWithChannelSite = await attachChannelSite(release);
const releaseId = await storeRelease(releaseWithChannelSite);
const releaseWithStudio = await attachStudio(release);
const releaseId = await storeRelease(releaseWithStudio);
return {
id: releaseId,

127
src/scrapers/bang.js Normal file
View File

@ -0,0 +1,127 @@
'use strict';
const bhttp = require('bhttp');
function encodeId(id) {
return Buffer
.from(id, 'hex')
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=/g, ',');
}
function scrapeLatest(scenes, site) {
return scenes.map(({ _source: scene }) => {
const release = {
site,
entryId: encodeId(scene.id),
title: scene.name,
description: scene.description,
actors: scene.actors.map(actor => actor.name),
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration,
};
const slug = release.title.toLowerCase().trim().replace(/\s+/g, '-');
release.url = `https://www.bang.com/video/${release.entryId}/${slug}`;
const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const photoset = scene.screenshots.filter(photo => photo.default === false);
const photos = defaultPoster ? photoset : photoset.slice(1);
const poster = defaultPoster || photoset[0];
release.poster = `https://i.bang.com/screenshots/${scene.dvd.id}/movie/1/${poster.screenId}.jpg`;
release.photos = photos.map(photo => `https://i.bang.com/screenshots/${scene.dvd.id}/movie/1/${photo.screenId}.jpg`);
release.trailer = {
src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`,
};
release.studio = scene.series.name
.replace(/[! .]/g, '')
.replace('&', 'and');
return release;
});
}
async function fetchLatest(site, page = 1) {
const clusterId = '617fb597b659459bafe6472470d9073a';
const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI=';
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
from: (page - 1) * 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
range: {
releaseDate: {
lte: 'now',
},
},
},
{
nested: {
path: 'studio',
query: {
bool: {
must: [
{
match: {
'studio.name': {
operator: 'AND',
query: 'bang! originals',
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeLatest(res.body.hits.hits, site);
}
module.exports = {
fetchLatest,
// fetchScene,
};

View File

@ -4,9 +4,6 @@ const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' ');
@ -102,16 +99,8 @@ async function scrapeScene(html, url, site, useGallery) {
const trailer = data.clip.qualities.find(clip => clip.quality === 'vga' || clip.quality === 'hd');
const studioName = $('.watchpage-studioname').first().text().trim();
const studioSlug = studioName.replace(/\s+/g, '').toLowerCase();
const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [studio, tags] = await Promise.all([
knex('studios')
.where({ name: studioName })
.orWhere({ slug: studioSlug })
.first(),
matchTags(rawTags),
]);
const studio = studioName.replace(/[\s.']+/g, '').toLowerCase();
const tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
return {
url,

View File

@ -2,6 +2,7 @@
// releases
const twentyonesextury = require('./21sextury');
const bang = require('./bang');
const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const dogfart = require('./dogfart');
@ -31,6 +32,7 @@ const pornhub = require('./pornhub');
module.exports = {
releases: {
'21sextury': twentyonesextury,
bang,
bangbros,
blowpass,
brazzers,

View File

@ -3,7 +3,7 @@
const path = require('path');
const Promise = require('bluebird');
const fs = require('fs-extra');
const fetchScene = require('../scrape-release');
const fetchScene = require('../scrape-releases');
const argv = require('../argv');