Added Bang! scraper. Showing studio on tile. Added favicons to tiles.

This commit is contained in:
ThePendulum 2020-01-06 05:19:38 +01:00
parent 3a7aa58abb
commit bb5b8085f6
90 changed files with 430 additions and 45 deletions

View File

@ -68,8 +68,16 @@ import Sites from '../sites/sites.vue';
async function fetchNetwork() { async function fetchNetwork() {
this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug); this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug);
if (this.network.studios) {
this.studios = this.network.studios.map(studio => ({
...studio,
network: this.network,
}));
}
this.sites = this.network.sites this.sites = this.network.sites
.filter(site => !site.independent) .filter(site => !site.independent)
.concat(this.studios)
.sort(({ name: nameA }, { name: nameB }) => nameA.localeCompare(nameB)); .sort(({ name: nameA }, { name: nameB }) => nameA.localeCompare(nameB));
this.releases = this.network.sites.map(site => site.releases).flat(); this.releases = this.network.sites.map(site => site.releases).flat();
@ -89,7 +97,8 @@ export default {
data() { data() {
return { return {
network: null, network: null,
sites: null, sites: [],
studios: [],
releases: [], releases: [],
pageTitle: null, pageTitle: null,
expanded: false, expanded: false,
@ -158,12 +167,13 @@ export default {
} }
.header { .header {
background: $profile;
width: 100%; width: 100%;
height: 4rem; height: 3rem;
display: none; display: none;
flex-shrink: 0;
justify-content: center; justify-content: center;
border-bottom: solid 1px $shadow-hint; border-bottom: solid 1px $shadow-hint;
background: $profile;
} }
.sites.compact { .sites.compact {

View File

@ -29,10 +29,18 @@ export default {
</script> </script>
<style lang="scss" scoped> <style lang="scss" scoped>
@import 'theme';
.networks { .networks {
display: grid; display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr)); grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem; grid-gap: 1rem;
padding: 1rem; padding: 1rem;
} }
@media(max-width: $breakpoint) {
.networks {
grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr));
}
}
</style> </style>

View File

@ -80,7 +80,21 @@
<span class="chain">presents</span> <span class="chain">presents</span>
<a :href="`/site/${release.site.slug}`"> <a
v-if="release.studio"
:href="`/site/${release.studio.slug}`"
>
<img
:src="`/img/logos/${release.network.slug}/${release.studio.slug}.png`"
:title="release.studio.name"
class="logo logo-site"
>
</a>
<a
v-else
:href="`/site/${release.site.slug}`"
>
<img <img
:src="`/img/logos/${release.network.slug}/${release.site.slug}.png`" :src="`/img/logos/${release.network.slug}/${release.site.slug}.png`"
:title="release.site.name" :title="release.site.name"

View File

@ -11,12 +11,24 @@
class="site site-link" class="site site-link"
>{{ release.network.name }}</router-link> >{{ release.network.name }}</router-link>
<router-link
v-else-if="release.studio"
:to="`/network/${release.network.slug}`"
class="site site-link"
><img
:src="`/img/logos/${release.network.slug}/favicon.png`"
class="favicon"
>{{ release.studio.name }}</router-link>
<router-link <router-link
v-else v-else
v-tooltip.bottom="`Part of ${release.network.name}`" v-tooltip.bottom="`Part of ${release.network.name}`"
:title="`Part of ${release.network.name}`" :title="`Part of ${release.network.name}`"
:to="`/site/${release.site.slug}`" :to="`/site/${release.site.slug}`"
class="site site-link" class="site site-link"
><img
:src="`/img/logos/${release.network.slug}/favicon.png`"
class="favicon"
>{{ release.site.name }}</router-link> >{{ release.site.name }}</router-link>
<a <a
@ -178,11 +190,18 @@ export default {
justify-content: space-between; justify-content: space-between;
position: absolute; position: absolute;
font-size: 0; font-size: 0;
.favicon {
height: 1rem;
margin: 0 .25rem 0 0;
}
} }
.site, .site,
.date { .date {
color: #fff; color: #fff;
display: flex;
align-items: center;
background: $shadow; background: $shadow;
position: relative; position: relative;
font-size: .8rem; font-size: .8rem;

View File

@ -37,6 +37,10 @@ function curateSite(site, network) {
if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release)); if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release));
if (site.network || network) curatedSite.network = site.network || network; if (site.network || network) curatedSite.network = site.network || network;
if (site.parameters) {
curatedSite.independent = !!JSON.parse(site.parameters).independent;
}
return curatedSite; return curatedSite;
} }
@ -52,6 +56,14 @@ function curateNetwork(network) {
curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork)); curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
} }
if (network.studios) {
curatedNetwork.studios = network.studios;
}
if (network.parameters) {
curatedNetwork.parameters = JSON.parse(network.parameters);
}
return curatedNetwork; return curatedNetwork;
} }

View File

@ -113,6 +113,12 @@ const releasesFragment = `
${releaseTagsFragment} ${releaseTagsFragment}
${releasePosterFragment} ${releasePosterFragment}
${siteFragment} ${siteFragment}
studio {
id
name
slug
url
}
} }
`; `;

View File

@ -22,6 +22,7 @@ function initNetworksActions(store, _router) {
slug slug
url url
${releasesFragment} ${releasesFragment}
parameters
network { network {
id id
name name
@ -29,6 +30,12 @@ function initNetworksActions(store, _router) {
url url
} }
} }
studios {
id
name
slug
url
}
} }
} }
`, { `, {

View File

@ -51,7 +51,7 @@ module.exports = {
'vixen', 'vixen',
'xempire', 'xempire',
], ],
fetchAfter: [3, 'months'], fetchAfter: [1, 'week'],
columns: [ columns: [
{ {
value: 'date', value: 'date',

51
package-lock.json generated
View File

@ -2626,6 +2626,14 @@
"toidentifier": "1.0.0" "toidentifier": "1.0.0"
} }
}, },
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"mime-db": { "mime-db": {
"version": "1.40.0", "version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
@ -4648,6 +4656,17 @@
"chardet": "^0.7.0", "chardet": "^0.7.0",
"iconv-lite": "^0.4.24", "iconv-lite": "^0.4.24",
"tmp": "^0.0.33" "tmp": "^0.0.33"
},
"dependencies": {
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"dev": true,
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
}
} }
}, },
"extglob": { "extglob": {
@ -6217,9 +6236,9 @@
} }
}, },
"iconv-lite": { "iconv-lite": {
"version": "0.4.24", "version": "0.5.0",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "integrity": "sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==",
"requires": { "requires": {
"safer-buffer": ">= 2.1.2 < 3" "safer-buffer": ">= 2.1.2 < 3"
} }
@ -7746,6 +7765,14 @@
"ms": "^2.1.1" "ms": "^2.1.1"
} }
}, },
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"ms": { "ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -9236,6 +9263,14 @@
"toidentifier": "1.0.0" "toidentifier": "1.0.0"
} }
}, },
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"setprototypeof": { "setprototypeof": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@ -12039,6 +12074,16 @@
"integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==",
"requires": { "requires": {
"iconv-lite": "0.4.24" "iconv-lite": "0.4.24"
},
"dependencies": {
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
}
} }
}, },
"whatwg-mimetype": { "whatwg-mimetype": {

View File

@ -85,6 +85,7 @@
"face-api.js": "^0.21.0", "face-api.js": "^0.21.0",
"fs-extra": "^7.0.1", "fs-extra": "^7.0.1",
"graphile-utils": "^4.5.6", "graphile-utils": "^4.5.6",
"iconv-lite": "^0.5.0",
"jsdom": "^15.2.1", "jsdom": "^15.2.1",
"knex": "^0.16.5", "knex": "^0.16.5",
"knex-migrate": "^1.7.4", "knex-migrate": "^1.7.4",

View File

@ -149,9 +149,17 @@
position: absolute; position: absolute;
font-size: 0; font-size: 0;
} }
.details .favicon[data-v-3abcf101] {
height: 1rem;
margin: 0 .25rem 0 0;
}
.site[data-v-3abcf101], .site[data-v-3abcf101],
.date[data-v-3abcf101] { .date[data-v-3abcf101] {
color: #fff; color: #fff;
display: -webkit-box;
display: flex;
-webkit-box-align: center;
align-items: center;
background: rgba(0, 0, 0, 0.5); background: rgba(0, 0, 0, 0.5);
position: relative; position: relative;
font-size: .8rem; font-size: .8rem;
@ -684,13 +692,14 @@
filter: drop-shadow(1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(-1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 1px 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 -1px 0 rgba(255, 255, 255, 0.2)); filter: drop-shadow(1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(-1px 0 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 1px 0 rgba(255, 255, 255, 0.2)) drop-shadow(0 -1px 0 rgba(255, 255, 255, 0.2));
} }
.header[data-v-e2e12602] { .header[data-v-e2e12602] {
background: #222;
width: 100%; width: 100%;
height: 4rem; height: 3rem;
display: none; display: none;
flex-shrink: 0;
-webkit-box-pack: center; -webkit-box-pack: center;
justify-content: center; justify-content: center;
border-bottom: solid 1px rgba(0, 0, 0, 0.1); border-bottom: solid 1px rgba(0, 0, 0, 0.1);
background: #222;
} }
.sites.compact[data-v-e2e12602] { .sites.compact[data-v-e2e12602] {
display: none; display: none;
@ -786,12 +795,18 @@
margin: 0; margin: 0;
} }
/* $primary: #ff886c; */
.networks[data-v-4709d404] { .networks[data-v-4709d404] {
display: grid; display: grid;
grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr)); grid-template-columns: repeat(auto-fit, minmax(15rem, 1fr));
grid-gap: 1rem; grid-gap: 1rem;
padding: 1rem; padding: 1rem;
} }
@media (max-width: 720px) {
.networks[data-v-4709d404] {
grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr));
}
}
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.photos[data-v-0a0430c7] { .photos[data-v-0a0430c7] {

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 948 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 882 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 165 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 101 KiB

After

Width:  |  Height:  |  Size: 101 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 90 KiB

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.6 KiB

After

Width:  |  Height:  |  Size: 5.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 547 KiB

After

Width:  |  Height:  |  Size: 105 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.6 KiB

After

Width:  |  Height:  |  Size: 7.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 95 KiB

After

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 104 KiB

After

Width:  |  Height:  |  Size: 104 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 240 KiB

After

Width:  |  Height:  |  Size: 240 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 124 KiB

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.2 KiB

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 139 KiB

After

Width:  |  Height:  |  Size: 139 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 85 KiB

After

Width:  |  Height:  |  Size: 85 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

View File

@ -8,6 +8,11 @@ const networks = [
url: 'https://www.21sextury.com', url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.', description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
}, },
{
slug: 'bang',
name: 'Bang!',
url: 'https://bang.com',
},
{ {
slug: 'bangbros', slug: 'bangbros',
name: 'Bang Bros', name: 'Bang Bros',

View File

@ -119,6 +119,15 @@ function getSites(networksMap) {
name: 'Sweet Sophie Moone', name: 'Sweet Sophie Moone',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
}, },
// BANG
{
slug: 'bang',
network_id: networksMap.bang,
name: 'Bang!',
url: 'https://bang.com',
description: null,
parameters: JSON.stringify({ independent: true }),
},
// BANGBROS // BANGBROS
{ {
slug: 'assparade', slug: 'assparade',
@ -638,7 +647,7 @@ function getSites(networksMap) {
}, },
{ {
slug: 'milfslikeitbig', slug: 'milfslikeitbig',
name: 'Milfs Like it Big', name: 'MILFs Like It Big',
url: 'https://www.brazzers.com/sites/view/id/36/milfs-like-it-big', url: 'https://www.brazzers.com/sites/view/id/36/milfs-like-it-big',
description: "When hubby's away milfy will play. These bored housewives want to get fucked and they want it now. They're experienced and know what they want. America's suburbs are full of these cum-de-sacs just waiting to get laid. Their round tits and thick asses are just begging for it. Cum inside, but don't park out front!", description: "When hubby's away milfy will play. These bored housewives want to get fucked and they want it now. They're experienced and know what they want. America's suburbs are full of these cum-de-sacs just waiting to get laid. Their round tits and thick asses are just begging for it. Cum inside, but don't park out front!",
network_id: networksMap.brazzers, network_id: networksMap.brazzers,
@ -2279,6 +2288,14 @@ function getSites(networksMap) {
slug: 'girlsofnaked', slug: 'girlsofnaked',
network_id: networksMap.realitykings, network_id: networksMap.realitykings,
}, },
{
name: 'Lil Humpers',
url: 'https://lilhumpers.com',
description: '',
parameters: JSON.stringify({ siteId: 310 }),
slug: 'lilhumpers',
network_id: networksMap.realitykings,
},
{ {
name: 'Mike in Brazil', name: 'Mike in Brazil',
url: 'https://www.realitykings.com/scenes?site=24', url: 'https://www.realitykings.com/scenes?site=24',

View File

@ -2,6 +2,91 @@ const upsert = require('../src/utils/upsert');
function getStudios(networksMap) { function getStudios(networksMap) {
return [ return [
// Bang!
{
name: 'Trickery',
slug: 'bangtrickery',
url: 'https://www.bang.com/original/4800/bang-trickery',
network_id: networksMap.bang,
},
{
name: 'Yngr',
slug: 'yngrcom',
url: 'https://www.bang.com/original/5010/bang-yngr',
network_id: networksMap.bang,
},
{
name: 'Roadside XXX',
slug: 'bangroadsidexxx',
url: 'https://www.bang.com/original/4864/roadside-xxx',
network_id: networksMap.bang,
},
{
name: 'Surprise',
slug: 'bangsurprise',
url: 'https://www.bang.com/original/5000/bang-surprise',
network_id: networksMap.bang,
},
{
name: 'Real Teens',
slug: 'bangrealteens',
url: 'https://www.bang.com/original/3366/bang-real-teens',
network_id: networksMap.bang,
},
{
name: 'FCK.news',
slug: 'bangfcknews',
url: 'https://www.bang.com/original/4998/bang-fckNews',
network_id: networksMap.bang,
},
{
name: 'Pretty & Raw',
slug: 'prettyandraw',
url: 'https://www.bang.com/original/4792/bang-pretty-and-raw',
network_id: networksMap.bang,
},
{
name: 'Japan',
slug: 'bangjapan',
url: 'https://www.bang.com/original/3079/bang-japan',
network_id: networksMap.bang,
},
{
name: 'Rammed',
slug: 'bangrammed',
url: 'https://www.bang.com/original/4836/bang-rammed',
network_id: networksMap.bang,
},
{
name: 'Glamkore',
slug: 'bangglamkore',
url: 'https://www.bang.com/original/4586/bang-glamkore',
network_id: networksMap.bang,
},
{
name: 'Screw The Cops',
slug: 'bangscrewthecops',
url: 'https://www.bang.com/original/4710/bang-screw-cops',
network_id: networksMap.bang,
},
{
name: 'Real MILFs',
slug: 'bangrealmilfs',
url: 'https://www.bang.com/original/4448/bang-real-milfs',
network_id: networksMap.bang,
},
{
name: 'Confessions',
slug: 'bangconfessions',
url: 'https://www.bang.com/original/4308/bang-confessions',
network_id: networksMap.bang,
},
{
name: 'Casting',
slug: 'bangcasting',
url: 'https://www.bang.com/original/3261/bang-casting',
network_id: networksMap.bang,
},
// LegalPorno // LegalPorno
{ {
slug: 'gonzocom', slug: 'gonzocom',

View File

@ -291,6 +291,6 @@ exports.seed = knex => Promise.resolve()
return Promise.all([ return Promise.all([
upsert('tags_posters', tagPosterEntries, 'tag_id', knex), upsert('tags_posters', tagPosterEntries, 'tag_id', knex),
upsert('tags_photos', tagPhotoEntries, 'tag_id', knex), upsert('tags_photos', tagPhotoEntries, ['tag_id', 'media_id'], knex),
]); ]);
}); });

View File

@ -1646,6 +1646,7 @@ const countries = [
}, },
{ {
name: 'United Kingdom of Great Britain and Northern Ireland', name: 'United Kingdom of Great Britain and Northern Ireland',
alias: 'United Kingdom',
code: 826, code: 826,
alpha2: 'GB', alpha2: 'GB',
alpha3: 'GBR', alpha3: 'GBR',

View File

@ -180,6 +180,23 @@ async function attachChannelSite(release) {
}; };
} }
async function attachStudio(release) {
if (!release.studio) {
return release;
}
const studio = await knex('studios')
.where('name', release.studio)
.orWhere('slug', release.studio)
.orWhere('url', release.studio)
.first();
return {
...release,
studio,
};
}
async function curateReleaseEntry(release) { async function curateReleaseEntry(release) {
const curatedRelease = { const curatedRelease = {
site_id: release.site.id, site_id: release.site.id,
@ -295,27 +312,31 @@ async function storeReleaseAssets(release, releaseId) {
await createMediaDirectory('releases', subpath); await createMediaDirectory('releases', subpath);
try { try {
await Promise.all([ // don't use Promise.all to prevent concurrency issues with duplicate detection
storePhotos(release.photos, { if (release.poster) {
targetId: releaseId, await storePhotos([release.poster], {
subpath,
primaryRole: release.poster ? null : 'poster',
}, identifier),
release.poster && storePhotos([release.poster], {
role: 'poster', role: 'poster',
targetId: releaseId, targetId: releaseId,
subpath, subpath,
}, identifier), }, identifier);
storePhotos(release.covers, { }
role: 'cover',
targetId: releaseId, await storePhotos(release.photos, {
subpath, targetId: releaseId,
}, identifier), subpath,
storeTrailer(release.trailer, { primaryRole: release.poster ? null : 'poster',
targetId: releaseId, }, identifier);
subpath,
}, identifier), await storePhotos(release.covers, {
]); role: 'cover',
targetId: releaseId,
subpath,
}, identifier);
await storeTrailer(release.trailer, {
targetId: releaseId,
subpath,
}, identifier);
} catch (error) { } catch (error) {
console.log(release.url, error); console.log(release.url, error);
} }
@ -363,7 +384,8 @@ async function storeReleases(releases) {
const storedReleases = await Promise.map(releases, async (release) => { const storedReleases = await Promise.map(releases, async (release) => {
try { try {
const releaseWithChannelSite = await attachChannelSite(release); const releaseWithChannelSite = await attachChannelSite(release);
const releaseId = await storeRelease(releaseWithChannelSite); const releaseWithStudio = await attachStudio(release);
const releaseId = await storeRelease(releaseWithStudio);
return { return {
id: releaseId, id: releaseId,

127
src/scrapers/bang.js Normal file
View File

@ -0,0 +1,127 @@
'use strict';
const bhttp = require('bhttp');
function encodeId(id) {
return Buffer
.from(id, 'hex')
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=/g, ',');
}
function scrapeLatest(scenes, site) {
return scenes.map(({ _source: scene }) => {
const release = {
site,
entryId: encodeId(scene.id),
title: scene.name,
description: scene.description,
actors: scene.actors.map(actor => actor.name),
tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration,
};
const slug = release.title.toLowerCase().trim().replace(/\s+/g, '-');
release.url = `https://www.bang.com/video/${release.entryId}/${slug}`;
const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay');
const defaultPoster = scene.screenshots.find(photo => photo.default === true);
const photoset = scene.screenshots.filter(photo => photo.default === false);
const photos = defaultPoster ? photoset : photoset.slice(1);
const poster = defaultPoster || photoset[0];
release.poster = `https://i.bang.com/screenshots/${scene.dvd.id}/movie/1/${poster.screenId}.jpg`;
release.photos = photos.map(photo => `https://i.bang.com/screenshots/${scene.dvd.id}/movie/1/${photo.screenId}.jpg`);
release.trailer = {
src: `https://i.bang.com/v/${scene.dvd.id}/${scene.identifier}/preview.mp4`,
};
release.studio = scene.series.name
.replace(/[! .]/g, '')
.replace('&', 'and');
return release;
});
}
async function fetchLatest(site, page = 1) {
const clusterId = '617fb597b659459bafe6472470d9073a';
const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI=';
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
from: (page - 1) * 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
range: {
releaseDate: {
lte: 'now',
},
},
},
{
nested: {
path: 'studio',
query: {
bool: {
must: [
{
match: {
'studio.name': {
operator: 'AND',
query: 'bang! originals',
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeLatest(res.body.hits.hits, site);
}
module.exports = {
fetchLatest,
// fetchScene,
};

View File

@ -4,9 +4,6 @@ const bhttp = require('bhttp');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const moment = require('moment'); const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
function extractTitle(originalTitle) { function extractTitle(originalTitle) {
const titleComponents = originalTitle.split(' '); const titleComponents = originalTitle.split(' ');
@ -102,16 +99,8 @@ async function scrapeScene(html, url, site, useGallery) {
const trailer = data.clip.qualities.find(clip => clip.quality === 'vga' || clip.quality === 'hd'); const trailer = data.clip.qualities.find(clip => clip.quality === 'vga' || clip.quality === 'hd');
const studioName = $('.watchpage-studioname').first().text().trim(); const studioName = $('.watchpage-studioname').first().text().trim();
const studioSlug = studioName.replace(/\s+/g, '').toLowerCase(); const studio = studioName.replace(/[\s.']+/g, '').toLowerCase();
const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [studio, tags] = await Promise.all([
knex('studios')
.where({ name: studioName })
.orWhere({ slug: studioSlug })
.first(),
matchTags(rawTags),
]);
return { return {
url, url,

View File

@ -2,6 +2,7 @@
// releases // releases
const twentyonesextury = require('./21sextury'); const twentyonesextury = require('./21sextury');
const bang = require('./bang');
const bangbros = require('./bangbros'); const bangbros = require('./bangbros');
const blowpass = require('./blowpass'); const blowpass = require('./blowpass');
const dogfart = require('./dogfart'); const dogfart = require('./dogfart');
@ -31,6 +32,7 @@ const pornhub = require('./pornhub');
module.exports = { module.exports = {
releases: { releases: {
'21sextury': twentyonesextury, '21sextury': twentyonesextury,
bang,
bangbros, bangbros,
blowpass, blowpass,
brazzers, brazzers,

View File

@ -3,7 +3,7 @@
const path = require('path'); const path = require('path');
const Promise = require('bluebird'); const Promise = require('bluebird');
const fs = require('fs-extra'); const fs = require('fs-extra');
const fetchScene = require('../scrape-release'); const fetchScene = require('../scrape-releases');
const argv = require('../argv'); const argv = require('../argv');