diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 9ecc9c62..5b4d9530 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -25,10 +25,11 @@ exports.up = knex => Promise.resolve() table.integer('index'); table.string('mime'); + table.string('hash'); table.string('type'); table.string('quality', 6); + table.float('entropy'); - table.string('hash'); table.text('comment'); table.string('source', 1000); diff --git a/public/img/logos/score/18eighteen.png b/public/img/logos/score/18eighteen.png index 708e0809..92a18913 100644 Binary files a/public/img/logos/score/18eighteen.png and b/public/img/logos/score/18eighteen.png differ diff --git a/public/img/logos/score/40somethingmag.png b/public/img/logos/score/40somethingmag.png index 556b0001..ea1ace7c 100644 Binary files a/public/img/logos/score/40somethingmag.png and b/public/img/logos/score/40somethingmag.png differ diff --git a/public/img/logos/score/50plusmilfs.png b/public/img/logos/score/50plusmilfs.png index 623ca3e6..ec868543 100644 Binary files a/public/img/logos/score/50plusmilfs.png and b/public/img/logos/score/50plusmilfs.png differ diff --git a/public/img/logos/score/60plusmilfs.png b/public/img/logos/score/60plusmilfs.png index b1a352eb..352edb9b 100644 Binary files a/public/img/logos/score/60plusmilfs.png and b/public/img/logos/score/60plusmilfs.png differ diff --git a/public/img/logos/score/ashleysageellison.png b/public/img/logos/score/ashleysageellison.png index 8dc03ccc..d1c58656 100644 Binary files a/public/img/logos/score/ashleysageellison.png and b/public/img/logos/score/ashleysageellison.png differ diff --git a/public/img/logos/score/autumnjade.png b/public/img/logos/score/autumnjade.png index 7289c2c7..45548e5a 100644 Binary files a/public/img/logos/score/autumnjade.png and b/public/img/logos/score/autumnjade.png differ diff --git a/public/img/logos/score/bigboobbundle.png b/public/img/logos/score/bigboobbundle.png index e7cca70a..68c7ff95 100644 Binary files a/public/img/logos/score/bigboobbundle.png and b/public/img/logos/score/bigboobbundle.png differ diff --git a/public/img/logos/score/bigboobspov.png b/public/img/logos/score/bigboobspov.png index f1b7593e..79904bab 100644 Binary files a/public/img/logos/score/bigboobspov.png and b/public/img/logos/score/bigboobspov.png differ diff --git a/public/img/logos/score/bigtitangelawhite.png b/public/img/logos/score/bigtitangelawhite.png index e10cec6a..57f75c13 100644 Binary files a/public/img/logos/score/bigtitangelawhite.png and b/public/img/logos/score/bigtitangelawhite.png differ diff --git a/public/img/logos/score/bigtithitomi.png b/public/img/logos/score/bigtithitomi.png index 4c925f5f..4e42656c 100644 Binary files a/public/img/logos/score/bigtithitomi.png and b/public/img/logos/score/bigtithitomi.png differ diff --git a/public/img/logos/score/bigtithooker.png b/public/img/logos/score/bigtithooker.png new file mode 100644 index 00000000..33931f20 Binary files /dev/null and b/public/img/logos/score/bigtithooker.png differ diff --git a/public/img/logos/score/bigtitterrynova.png b/public/img/logos/score/bigtitterrynova.png index 9cad638c..3cc504fb 100644 Binary files a/public/img/logos/score/bigtitterrynova.png and b/public/img/logos/score/bigtitterrynova.png differ diff --git a/public/img/logos/score/bigtitvenera.png b/public/img/logos/score/bigtitvenera.png index a4d9f5da..8312fda2 100644 Binary files a/public/img/logos/score/bigtitvenera.png and b/public/img/logos/score/bigtitvenera.png differ diff --git a/public/img/logos/score/blackandstacked.png b/public/img/logos/score/blackandstacked.png index 33593a4c..e759db8b 100644 Binary files a/public/img/logos/score/blackandstacked.png and b/public/img/logos/score/blackandstacked.png differ diff --git a/public/img/logos/score/bonedathome.png b/public/img/logos/score/bonedathome.png new file mode 100644 index 00000000..0e7a7697 Binary files /dev/null and b/public/img/logos/score/bonedathome.png differ diff --git a/public/img/logos/score/bootyliciousmag.png b/public/img/logos/score/bootyliciousmag.png index 121c4c55..e0ad9f2b 100644 Binary files a/public/img/logos/score/bootyliciousmag.png and b/public/img/logos/score/bootyliciousmag.png differ diff --git a/public/img/logos/score/bustyangelique.png b/public/img/logos/score/bustyangelique.png index 5459cb73..3fe4197c 100644 Binary files a/public/img/logos/score/bustyangelique.png and b/public/img/logos/score/bustyangelique.png differ diff --git a/public/img/logos/score/bustyarianna.png b/public/img/logos/score/bustyarianna.png new file mode 100644 index 00000000..1884b2ac Binary files /dev/null and b/public/img/logos/score/bustyarianna.png differ diff --git a/public/img/logos/score/bustydanniashe.png b/public/img/logos/score/bustydanniashe.png new file mode 100644 index 00000000..20ff7f01 Binary files /dev/null and b/public/img/logos/score/bustydanniashe.png differ diff --git a/public/img/logos/score/bustydustystash.png b/public/img/logos/score/bustydustystash.png index 90413066..85d8309a 100644 Binary files a/public/img/logos/score/bustydustystash.png and b/public/img/logos/score/bustydustystash.png differ diff --git a/public/img/logos/score/bustyinescudna.png b/public/img/logos/score/bustyinescudna.png new file mode 100644 index 00000000..c8d4ab3a Binary files /dev/null and b/public/img/logos/score/bustyinescudna.png differ diff --git a/public/img/logos/score/bustykellykay.png b/public/img/logos/score/bustykellykay.png new file mode 100644 index 00000000..59f64fd2 Binary files /dev/null and b/public/img/logos/score/bustykellykay.png differ diff --git a/public/img/logos/score/bustykerrymarie.png b/public/img/logos/score/bustykerrymarie.png new file mode 100644 index 00000000..e3286b09 Binary files /dev/null and b/public/img/logos/score/bustykerrymarie.png differ diff --git a/public/img/logos/score/bustylornamorgan.png b/public/img/logos/score/bustylornamorgan.png new file mode 100644 index 00000000..6cd86696 Binary files /dev/null and b/public/img/logos/score/bustylornamorgan.png differ diff --git a/public/img/logos/score/bustymerilyn.png b/public/img/logos/score/bustymerilyn.png new file mode 100644 index 00000000..172afe62 Binary files /dev/null and b/public/img/logos/score/bustymerilyn.png differ diff --git a/public/img/logos/score/bustyoldsluts.png b/public/img/logos/score/bustyoldsluts.png new file mode 100644 index 00000000..292eb0b6 Binary files /dev/null and b/public/img/logos/score/bustyoldsluts.png differ diff --git a/public/img/logos/score/bustysammieblack.png b/public/img/logos/score/bustysammieblack.png new file mode 100644 index 00000000..e86a2a10 Binary files /dev/null and b/public/img/logos/score/bustysammieblack.png differ diff --git a/public/img/logos/score/cherrybrady.png b/public/img/logos/score/cherrybrady.png new file mode 100644 index 00000000..fd270d85 Binary files /dev/null and b/public/img/logos/score/cherrybrady.png differ diff --git a/public/img/logos/score/chloesworld.png b/public/img/logos/score/chloesworld.png new file mode 100644 index 00000000..dfef87dd Binary files /dev/null and b/public/img/logos/score/chloesworld.png differ diff --git a/public/img/logos/score/christymarks.png b/public/img/logos/score/christymarks.png new file mode 100644 index 00000000..db25d708 Binary files /dev/null and b/public/img/logos/score/christymarks.png differ diff --git a/public/img/logos/score/creampieforgranny.png b/public/img/logos/score/creampieforgranny.png new file mode 100644 index 00000000..46a218fe Binary files /dev/null and b/public/img/logos/score/creampieforgranny.png differ diff --git a/public/img/logos/score/crystalgunnsworld.png b/public/img/logos/score/crystalgunnsworld.png new file mode 100644 index 00000000..38257976 Binary files /dev/null and b/public/img/logos/score/crystalgunnsworld.png differ diff --git a/public/img/logos/score/daylenerio.png b/public/img/logos/score/daylenerio.png new file mode 100644 index 00000000..cb72944b Binary files /dev/null and b/public/img/logos/score/daylenerio.png differ diff --git a/public/img/logos/score/desiraesworld.png b/public/img/logos/score/desiraesworld.png new file mode 100644 index 00000000..2bba2467 Binary files /dev/null and b/public/img/logos/score/desiraesworld.png differ diff --git a/public/img/logos/score/dianepoppos.png b/public/img/logos/score/dianepoppos.png new file mode 100644 index 00000000..8b0152fc Binary files /dev/null and b/public/img/logos/score/dianepoppos.png differ diff --git a/public/img/logos/score/evanottyvideos.png b/public/img/logos/score/evanottyvideos.png new file mode 100644 index 00000000..95b56366 Binary files /dev/null and b/public/img/logos/score/evanottyvideos.png differ diff --git a/public/img/logos/score/favicon.png b/public/img/logos/score/favicon.png index c12d2687..6f0b2c17 100644 Binary files a/public/img/logos/score/favicon.png and b/public/img/logos/score/favicon.png differ diff --git a/public/img/logos/score/feedherfuckher.png b/public/img/logos/score/feedherfuckher.png new file mode 100644 index 00000000..834b9f5b Binary files /dev/null and b/public/img/logos/score/feedherfuckher.png differ diff --git a/public/img/logos/score/flatandfuckedmilfs.png b/public/img/logos/score/flatandfuckedmilfs.png new file mode 100644 index 00000000..94240170 Binary files /dev/null and b/public/img/logos/score/flatandfuckedmilfs.png differ diff --git a/public/img/logos/score/grannygetsafacial.png b/public/img/logos/score/grannygetsafacial.png new file mode 100644 index 00000000..a2a94874 Binary files /dev/null and b/public/img/logos/score/grannygetsafacial.png differ diff --git a/public/img/logos/score/grannylovesbbc.png b/public/img/logos/score/grannylovesbbc.png new file mode 100644 index 00000000..1020c9c9 Binary files /dev/null and b/public/img/logos/score/grannylovesbbc.png differ diff --git a/public/img/logos/score/grannylovesyoungcock.png b/public/img/logos/score/grannylovesyoungcock.png new file mode 100644 index 00000000..133b2b1d Binary files /dev/null and b/public/img/logos/score/grannylovesyoungcock.png differ diff --git a/public/img/logos/score/homealonemilfs.png b/public/img/logos/score/homealonemilfs.png new file mode 100644 index 00000000..622a54c2 Binary files /dev/null and b/public/img/logos/score/homealonemilfs.png differ diff --git a/public/img/logos/score/ibonedyourmom.png b/public/img/logos/score/ibonedyourmom.png new file mode 100644 index 00000000..c5b3fca2 Binary files /dev/null and b/public/img/logos/score/ibonedyourmom.png differ diff --git a/public/img/logos/score/ifuckedtheboss.png b/public/img/logos/score/ifuckedtheboss.png new file mode 100644 index 00000000..3cc63baa Binary files /dev/null and b/public/img/logos/score/ifuckedtheboss.png differ diff --git a/public/img/logos/score/jessicaturner.png b/public/img/logos/score/jessicaturner.png new file mode 100644 index 00000000..714f223d Binary files /dev/null and b/public/img/logos/score/jessicaturner.png differ diff --git a/public/img/logos/score/joanabliss.png b/public/img/logos/score/joanabliss.png index 49de0954..c6ae54b2 100644 Binary files a/public/img/logos/score/joanabliss.png and b/public/img/logos/score/joanabliss.png differ diff --git a/public/img/logos/score/juliamiles.png b/public/img/logos/score/juliamiles.png new file mode 100644 index 00000000..561da059 Binary files /dev/null and b/public/img/logos/score/juliamiles.png differ diff --git a/public/img/logos/score/karinahart.png b/public/img/logos/score/karinahart.png new file mode 100644 index 00000000..7dfbdfc6 Binary files /dev/null and b/public/img/logos/score/karinahart.png differ diff --git a/public/img/logos/score/karlajames.png b/public/img/logos/score/karlajames.png new file mode 100644 index 00000000..2c963c82 Binary files /dev/null and b/public/img/logos/score/karlajames.png differ diff --git a/public/img/logos/score/leannecrowvideos.png b/public/img/logos/score/leannecrowvideos.png new file mode 100644 index 00000000..b64f6a7e Binary files /dev/null and b/public/img/logos/score/leannecrowvideos.png differ diff --git a/public/img/logos/score/legsex.png b/public/img/logos/score/legsex.png new file mode 100644 index 00000000..fe6d8e72 Binary files /dev/null and b/public/img/logos/score/legsex.png differ diff --git a/public/img/logos/score/linseysworld.png b/public/img/logos/score/linseysworld.png new file mode 100644 index 00000000..5dadbd28 Binary files /dev/null and b/public/img/logos/score/linseysworld.png differ diff --git a/public/img/logos/score/megatitsminka.png b/public/img/logos/score/megatitsminka.png new file mode 100644 index 00000000..294ff287 Binary files /dev/null and b/public/img/logos/score/megatitsminka.png differ diff --git a/public/img/logos/score/mickybells.png b/public/img/logos/score/mickybells.png new file mode 100644 index 00000000..8b3e8892 Binary files /dev/null and b/public/img/logos/score/mickybells.png differ diff --git a/public/img/logos/score/milfbundle.png b/public/img/logos/score/milfbundle.png new file mode 100644 index 00000000..b3d206c7 Binary files /dev/null and b/public/img/logos/score/milfbundle.png differ diff --git a/public/img/logos/score/milfthreesomes.png b/public/img/logos/score/milfthreesomes.png new file mode 100644 index 00000000..bd516014 Binary files /dev/null and b/public/img/logos/score/milfthreesomes.png differ diff --git a/public/img/logos/score/milftugs.png b/public/img/logos/score/milftugs.png new file mode 100644 index 00000000..d9223ce8 Binary files /dev/null and b/public/img/logos/score/milftugs.png differ diff --git a/public/img/logos/score/misc/joana-bliss.png b/public/img/logos/score/misc/joana-bliss.png new file mode 100644 index 00000000..679793d8 Binary files /dev/null and b/public/img/logos/score/misc/joana-bliss.png differ diff --git a/public/img/logos/score/misc/porn-mega-load.png b/public/img/logos/score/misc/porn-mega-load.png new file mode 100644 index 00000000..89084d7e Binary files /dev/null and b/public/img/logos/score/misc/porn-mega-load.png differ diff --git a/public/img/logos/score/nataliefiore.png b/public/img/logos/score/nataliefiore.png new file mode 100644 index 00000000..b99a2bf8 Binary files /dev/null and b/public/img/logos/score/nataliefiore.png differ diff --git a/public/img/logos/score/naughtyfootjobs.png b/public/img/logos/score/naughtyfootjobs.png new file mode 100644 index 00000000..99629fac Binary files /dev/null and b/public/img/logos/score/naughtyfootjobs.png differ diff --git a/public/img/logos/score/naughtymag.png b/public/img/logos/score/naughtymag.png new file mode 100644 index 00000000..0f712dc2 Binary files /dev/null and b/public/img/logos/score/naughtymag.png differ diff --git a/public/img/logos/score/naughtytugs.png b/public/img/logos/score/naughtytugs.png new file mode 100644 index 00000000..c019ab4f Binary files /dev/null and b/public/img/logos/score/naughtytugs.png differ diff --git a/public/img/logos/score/network.png b/public/img/logos/score/network.png index 067c5298..79846bc6 100644 Binary files a/public/img/logos/score/network.png and b/public/img/logos/score/network.png differ diff --git a/public/img/logos/score/nicolepeters.png b/public/img/logos/score/nicolepeters.png new file mode 100644 index 00000000..8cbbbf57 Binary files /dev/null and b/public/img/logos/score/nicolepeters.png differ diff --git a/public/img/logos/score/oldhornymilfs.png b/public/img/logos/score/oldhornymilfs.png new file mode 100644 index 00000000..a4f50e1e Binary files /dev/null and b/public/img/logos/score/oldhornymilfs.png differ diff --git a/public/img/logos/score/pickinguppussy.png b/public/img/logos/score/pickinguppussy.png new file mode 100644 index 00000000..465f3789 Binary files /dev/null and b/public/img/logos/score/pickinguppussy.png differ diff --git a/public/img/logos/score/pornloser.png b/public/img/logos/score/pornloser.png new file mode 100644 index 00000000..5e784f78 Binary files /dev/null and b/public/img/logos/score/pornloser.png differ diff --git a/public/img/logos/score/pornmegaload.png b/public/img/logos/score/pornmegaload.png new file mode 100644 index 00000000..39ad6463 Binary files /dev/null and b/public/img/logos/score/pornmegaload.png differ diff --git a/public/img/logos/score/sarennasworld.png b/public/img/logos/score/sarennasworld.png new file mode 100644 index 00000000..b3e5b773 Binary files /dev/null and b/public/img/logos/score/sarennasworld.png differ diff --git a/public/img/logos/score/scoreclassics.png b/public/img/logos/score/scoreclassics.png index 6c4249c7..ea287b4f 100644 Binary files a/public/img/logos/score/scoreclassics.png and b/public/img/logos/score/scoreclassics.png differ diff --git a/public/img/logos/score/scoreland.png b/public/img/logos/score/scoreland.png index 7b2a31a2..b3cff4ac 100644 Binary files a/public/img/logos/score/scoreland.png and b/public/img/logos/score/scoreland.png differ diff --git a/public/img/logos/score/scoreland2.png b/public/img/logos/score/scoreland2.png index c060e612..fa708a0d 100644 Binary files a/public/img/logos/score/scoreland2.png and b/public/img/logos/score/scoreland2.png differ diff --git a/public/img/logos/score/scorelandtv.png b/public/img/logos/score/scorelandtv.png index 563905a1..43db5df7 100644 Binary files a/public/img/logos/score/scorelandtv.png and b/public/img/logos/score/scorelandtv.png differ diff --git a/public/img/logos/score/scoretv.png b/public/img/logos/score/scoretv.png index 61dea998..662ee4e6 100644 Binary files a/public/img/logos/score/scoretv.png and b/public/img/logos/score/scoretv.png differ diff --git a/public/img/logos/score/scorevideos.png b/public/img/logos/score/scorevideos.png index 293caa92..6237f9d9 100644 Binary files a/public/img/logos/score/scorevideos.png and b/public/img/logos/score/scorevideos.png differ diff --git a/public/img/logos/score/sharizelvideos.png b/public/img/logos/score/sharizelvideos.png new file mode 100644 index 00000000..4b40fbbc Binary files /dev/null and b/public/img/logos/score/sharizelvideos.png differ diff --git a/public/img/logos/score/silversluts.png b/public/img/logos/score/silversluts.png new file mode 100644 index 00000000..0ca05be2 Binary files /dev/null and b/public/img/logos/score/silversluts.png differ diff --git a/public/img/logos/score/stacyvandenbergboobs.png b/public/img/logos/score/stacyvandenbergboobs.png new file mode 100644 index 00000000..24e3c053 Binary files /dev/null and b/public/img/logos/score/stacyvandenbergboobs.png differ diff --git a/public/img/logos/score/susiewildin.png b/public/img/logos/score/susiewildin.png new file mode 100644 index 00000000..799919a2 Binary files /dev/null and b/public/img/logos/score/susiewildin.png differ diff --git a/public/img/logos/score/tawnypeaks.png b/public/img/logos/score/tawnypeaks.png new file mode 100644 index 00000000..fe348c7c Binary files /dev/null and b/public/img/logos/score/tawnypeaks.png differ diff --git a/public/img/logos/score/thescoregroup.png b/public/img/logos/score/thescoregroup.png index c6c5d4e1..b6134d6b 100644 Binary files a/public/img/logos/score/thescoregroup.png and b/public/img/logos/score/thescoregroup.png differ diff --git a/public/img/logos/score/tiffanytowers.png b/public/img/logos/score/tiffanytowers.png new file mode 100644 index 00000000..8fff72e2 Binary files /dev/null and b/public/img/logos/score/tiffanytowers.png differ diff --git a/public/img/logos/score/titsandtugs.png b/public/img/logos/score/titsandtugs.png new file mode 100644 index 00000000..08ee0bc5 Binary files /dev/null and b/public/img/logos/score/titsandtugs.png differ diff --git a/public/img/logos/score/tnatryouts.png b/public/img/logos/score/tnatryouts.png new file mode 100644 index 00000000..070243b0 Binary files /dev/null and b/public/img/logos/score/tnatryouts.png differ diff --git a/public/img/logos/score/valoryirene.png b/public/img/logos/score/valoryirene.png new file mode 100644 index 00000000..cba7dd4c Binary files /dev/null and b/public/img/logos/score/valoryirene.png differ diff --git a/public/img/logos/score/xlgirls.png b/public/img/logos/score/xlgirls.png new file mode 100644 index 00000000..f2141104 Binary files /dev/null and b/public/img/logos/score/xlgirls.png differ diff --git a/public/img/logos/score/yourmomlovesanal.png b/public/img/logos/score/yourmomlovesanal.png index 185ade54..5246ee8b 100644 Binary files a/public/img/logos/score/yourmomlovesanal.png and b/public/img/logos/score/yourmomlovesanal.png differ diff --git a/public/img/logos/score/yourmomsgotbigtits.png b/public/img/logos/score/yourmomsgotbigtits.png new file mode 100644 index 00000000..21f944e1 Binary files /dev/null and b/public/img/logos/score/yourmomsgotbigtits.png differ diff --git a/public/img/logos/score/yourwifemymeat.png b/public/img/logos/score/yourwifemymeat.png new file mode 100644 index 00000000..74300daa Binary files /dev/null and b/public/img/logos/score/yourwifemymeat.png differ diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 76f2ec8c..083cd6f1 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -4481,6 +4481,8 @@ const sites = [ slug: 'milfbundle', url: 'https://www.milfbundle.com', network: 'score', + show: false, + scrape: false, }, { name: 'Teaming Cock', @@ -4543,10 +4545,12 @@ const sites = [ network: 'score', }, { - name: 'Definition Porn Videos', + name: 'Porn Mega Load', slug: 'pornmegaload', url: 'https://www.pornmegaload.com', network: 'score', + show: false, + scrape: false, }, { name: 'SaRennas World', @@ -4630,13 +4634,13 @@ const sites = [ }, { name: 'Tawny Peaks', - slug: 'tawny', + slug: 'tawnypeaks', url: 'https://www.bigboobbundle.com/tawny-peaks', network: 'score', }, { name: 'Tiffany Towers', - slug: 'tiffany', + slug: 'tiffanytowers', url: 'https://www.bigboobbundle.com/tiffany-towers', network: 'score', }, diff --git a/src/actors.js b/src/actors.js index a4e7fee9..2aa0e06b 100644 --- a/src/actors.js +++ b/src/actors.js @@ -12,7 +12,7 @@ const scrapers = require('./scrapers/scrapers'); const whereOr = require('./utils/where-or'); const resolvePlace = require('./utils/resolve-place'); const slugify = require('./utils/slugify'); -const { createMediaDirectory, storePhotos } = require('./media'); +const { createMediaDirectory, storePhotos } = require('./media_legacy'); async function curateActor(actor) { const [aliases, avatar, photos, social] = await Promise.all([ diff --git a/src/media.js b/src/media.js index a630efd7..ed52cabe 100644 --- a/src/media.js +++ b/src/media.js @@ -2,39 +2,32 @@ const config = require('config'); const Promise = require('bluebird'); -const path = require('path'); -const fs = require('fs-extra'); const bhttp = require('bhttp'); const mime = require('mime'); const sharp = require('sharp'); const blake2 = require('blake2'); -const logger = require('./logger')(__filename); +const logger = require('./logger'); const knex = require('./knex'); -const upsert = require('./utils/upsert'); -const { ex } = require('./utils/q'); function getHash(buffer) { const hash = blake2.createHash('blake2b', { digestLength: 24 }); - hash.update(buffer); return hash.digest('hex'); } -function pluckPhotos(photos, specifiedLimit) { +function pluckItems(items, specifiedLimit) { const limit = specifiedLimit || config.media.limit; - if (photos.length <= limit) { - return photos; - } + if (items.length <= limit) return items; const plucked = [1] .concat( - Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (photos.length / (limit - 1)))), + Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (items.length / (limit - 1)))), ); - return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close + return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close } async function getEntropy(buffer) { @@ -49,337 +42,74 @@ async function getEntropy(buffer) { } } -async function createThumbnail(buffer) { +async function fetchItem(source, index, existingItemsBySource, attempt = 1) { try { - const thumbnail = sharp(buffer) - .resize({ - height: config.media.thumbnailSize, - withoutEnlargement: true, - }) - .jpeg({ - quality: config.media.thumbnailQuality, - }) - .toBuffer(); - - return thumbnail; - } catch (error) { - logger.error(`Failed to create thumbnail: ${error.message}`); - } - - return null; -} - -async function createMediaDirectory(domain, subpath) { - const filepath = path.join(config.media.path, domain, subpath); - - await fs.mkdir(filepath, { recursive: true }); - return filepath; -} - -function curatePhotoEntries(files) { - return files.map((file, index) => ({ - path: file.filepath, - thumbnail: file.thumbpath, - mime: file.mimetype, - hash: file.hash, - source: file.source, - index, - })); -} - -async function findDuplicates(photos, identifier, prop = null) { - const duplicates = await knex('media') - .whereIn(identifier, photos.flat().map((photo) => { - if (prop) return photo[prop]; - if (photo.src) return photo.src; - - return photo; - })); - - const duplicateLookup = new Set(duplicates.map(photo => photo[prop || identifier])); - - const originals = photos.filter((source) => { if (Array.isArray(source)) { - return !source.some(sourceX => duplicateLookup.has((prop && sourceX[prop]) || (sourceX.src && sourceX))); + // fallbacks provided + return source.reduce((outcome, sourceX) => outcome.catch(async () => { + const item = await fetchItem(sourceX, index, existingItemsBySource); + + if (item) { + return item; + } + + throw new Error(`Item not available: ${source}`); + }), Promise.reject(new Error())); } - return !duplicateLookup.has((prop && source[prop]) || (source.src && source)); - }); + if (existingItemsBySource[source]) { + return existingItemsBySource[source]; + } - return [duplicates, originals]; -} - -async function extractPhoto(source) { - const res = await bhttp.get(source.src); - - if (res.statusCode === 200) { - const { q } = ex(res.body.toString()); - - return source.extract(q); - } - - return null; -} - -async function fetchPhoto(photoUrl, index, label, attempt = 1) { - if (photoUrl.src && photoUrl.extract) { - // source links to page containing a (presumably) tokenized photo - const photo = await extractPhoto(photoUrl); - - return fetchPhoto(photo, index, label); - } - - if (Array.isArray(photoUrl)) { - return photoUrl.reduce(async (outcome, url) => outcome.catch(async () => { - const photo = await fetchPhoto(url, index, label); - - if (photo) { - return photo; - } - - throw new Error('Photo not available'); - }), Promise.reject(new Error())); - } - - try { - const { pathname } = new URL(photoUrl); - const res = await bhttp.get(photoUrl); + const res = await bhttp.get(source); if (res.statusCode === 200) { + const { pathname } = new URL(source); const mimetype = mime.getType(pathname); const extension = mime.getExtension(mimetype); const hash = getHash(res.body); const entropy = await getEntropy(res.body); return { - photo: res.body, + file: res.body, mimetype, extension, hash, entropy, - source: photoUrl, + source, }; } throw new Error(`Response ${res.statusCode} not OK`); } catch (error) { - logger.warn(`Failed attempt ${attempt}/3 to fetch photo ${index + 1} for ${label} (${photoUrl}): ${error}`); - - if (attempt < 3) { - await Promise.delay(5000); - return fetchPhoto(photoUrl, index, label, attempt + 1); + if (attempt <= 3) { + return fetchItem(source, index, existingItemsBySource, attempt + 1); } - return null; + throw new Error(`Failed to fetch media from ${source}: ${error}`); } } -async function savePhotos(files, { - domain = 'release', - subpath, - role = 'photo', - naming = 'index', -}) { - return Promise.map(files, async (file, index) => { - try { - const timestamp = new Date().getTime(); - const thumbnail = await createThumbnail(file.photo); - - const filename = naming === 'index' - ? `${file.role || role}${index + 1}` - : `${timestamp + index}`; - - const filepath = path.join(`${domain}s`, subpath, `${filename}.${file.extension}`); - const thumbpath = path.join(`${domain}s`, subpath, `${filename}_thumb.${file.extension}`); - - await Promise.all([ - fs.writeFile(path.join(config.media.path, filepath), file.photo), - fs.writeFile(path.join(config.media.path, thumbpath), thumbnail), - ]); - - return { - ...file, - thumbnail, - filepath, - thumbpath, - }; - } catch (error) { - logger.error(`Failed to store ${domain} ${role} to ${subpath}: ${error.message}`); - return null; - } - }); +async function fetchItems(itemSources, existingItemsBySource) { + return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource)); } -async function storePhotos(photos, { - domain = 'release', - role = 'photo', - naming = 'index', - targetId, - subpath, - primaryRole, // role to assign to first photo if not already in database, used mainly for avatars - entropyFilter = 2.5, // filter out fallback avatars and other generic clipart -}, label) { - if (!photos || photos.length === 0) { - logger.info(`No ${role}s available for ${label}`); - return; - } +async function storeReleaseMedia(releases, { + type = 'poster', +} = {}) { + const pluckedSources = releases.map(release => pluckItems(release[type])); + const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat()); + const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {}); - const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration - const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label); + const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource); + const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash)); + const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {}); - logger.info(`Fetching ${sourceOriginals.length} new ${role}s, ${sourceDuplicates.length} already present by source for ${label}`); + const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]); - const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), { - concurrency: 10, - }).filter(photo => photo && photo.entropy > entropyFilter); - - const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks - const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label); - - logger.info(`Saving ${hashOriginals.length} new ${role}s, ${hashDuplicates.length} already present by hash for ${label}`); - - const savedPhotos = await savePhotos(hashOriginals, { - domain, - role, - targetId, - subpath, - naming, - }); - - const curatedPhotoEntries = curatePhotoEntries(savedPhotos.filter(Boolean), domain, role, targetId); - - const newPhotos = await knex('media').insert(curatedPhotoEntries).returning('*'); - const photoEntries = Array.isArray(newPhotos) - ? [...sourceDuplicates, ...hashDuplicates, ...newPhotos] - : [...sourceDuplicates, ...hashDuplicates]; - - const photoAssociations = photoEntries - .map(photoEntry => ({ - [`${domain}_id`]: targetId, - media_id: photoEntry.id, - })); - - if (primaryRole) { - // store one photo as a 'primary' photo, such as an avatar or cover - const primaryPhoto = await knex(`${domain}s_${primaryRole}s`) - .where(`${domain}_id`, targetId) - .first(); - - if (primaryPhoto) { - const remainingAssociations = photoAssociations.filter(association => association.media_id !== primaryPhoto.media_id); - - await upsert(`${domain}s_${role}s`, remainingAssociations, [`${domain}_id`, 'media_id']); - return; - } - - await Promise.all([ - upsert(`${domain}s_${primaryRole}s`, photoAssociations.slice(0, 1), [`${domain}_id`, 'media_id']), - upsert(`${domain}s_${role}s`, photoAssociations.slice(1), [`${domain}_id`, 'media_id']), - ]); - - return; - } - - await upsert(`${domain}s_${role}s`, photoAssociations, [`${domain}_id`, 'media_id']); -} - -/* -async function storeReleasePhotos(releases, label) { - const sources = releases.map(release => pluckPhotos(release.photos)).flat(); - const uniqueSources = Array.from(new Set(sources)); - - const [sourceDuplicates, sourceOriginals] = await findDuplicates(uniqueSources, 'source', null, label); - - const metaFiles = await Promise.map( - sourceOriginals, - async (photoUrl, index) => fetchPhoto(photoUrl, index, label), - { concurrency: 10 }, - ) - .filter(photo => photo); - - const hashUniques = Object.values(metaFiles.reduce((acc, file) => { - if (!acc[file.hash]) acc[file.hash] = file; - - return acc; - }, {})); - - const [hashDuplicates, hashOriginals] = await findDuplicates(hashUniques, 'hash', 'hash', label); - - const sourceHashes = metaFiles.concat(sourceDuplicates).reduce((acc, file) => { - acc[file.source] = file.hash; - return acc; - }, {}); - - const associations = releases.map(release => release.photos.map(source => [release.id, sourceHashes[source]])).flat(); - - console.log(associations); -} -*/ - -async function storeTrailer(trailers, { - domain = 'releases', - role = 'trailer', - targetId, - subpath, -}, label) { - // support scrapers supplying multiple qualities - const trailer = Array.isArray(trailers) - ? trailers.find(trailerX => config.media.trailerQuality.includes(trailerX.quality)) || trailers[0] - : trailers; - - if (!trailer || !trailer.src) { - logger.info(`No ${role} available for ${label}`); - return; - } - - const [sourceDuplicates, sourceOriginals] = await findDuplicates([trailer], 'source', 'src', label); - - const metaFiles = await Promise.map(sourceOriginals, async (trailerX) => { - const { pathname } = new URL(trailerX.src); - const mimetype = trailerX.type || mime.getType(pathname); - - const res = await bhttp.get(trailerX.src); - const hash = getHash(res.body); - const filepath = path.join(domain, subpath, `${role}${trailerX.quality ? `_${trailerX.quality}` : ''}.${mime.getExtension(mimetype)}`); - - return { - trailer: res.body, - path: filepath, - mime: mimetype, - source: trailerX.src, - quality: trailerX.quality || null, - hash, - }; - }); - - const [hashDuplicates, hashOriginals] = await findDuplicates(metaFiles, 'hash', 'hash', label); - - const newTrailers = await knex('media') - .insert(hashOriginals.map(trailerX => ({ - path: trailerX.path, - mime: trailerX.mime, - source: trailerX.source, - quality: trailerX.quality, - hash: trailerX.hash, - type: role, - }))) - .returning('*'); - - await Promise.all(hashOriginals.map(trailerX => fs.writeFile(path.join(config.media.path, trailerX.path), trailerX.trailer))); - - const trailerEntries = Array.isArray(newTrailers) - ? [...sourceDuplicates, ...hashDuplicates, ...newTrailers] - : [...sourceDuplicates, ...hashDuplicates]; - - await upsert(`releases_${role}s`, trailerEntries.map(trailerEntry => ({ - release_id: targetId, - media_id: trailerEntry.id, - })), ['release_id', 'media_id']); + console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems); } module.exports = { - createMediaDirectory, - storePhotos, - // storeReleasePhotos, - storeTrailer, + storeReleaseMedia, }; diff --git a/src/media_legacy.js b/src/media_legacy.js new file mode 100644 index 00000000..a630efd7 --- /dev/null +++ b/src/media_legacy.js @@ -0,0 +1,385 @@ +'use strict'; + +const config = require('config'); +const Promise = require('bluebird'); +const path = require('path'); +const fs = require('fs-extra'); +const bhttp = require('bhttp'); +const mime = require('mime'); +const sharp = require('sharp'); +const blake2 = require('blake2'); + +const logger = require('./logger')(__filename); +const knex = require('./knex'); +const upsert = require('./utils/upsert'); +const { ex } = require('./utils/q'); + +function getHash(buffer) { + const hash = blake2.createHash('blake2b', { digestLength: 24 }); + + hash.update(buffer); + + return hash.digest('hex'); +} + +function pluckPhotos(photos, specifiedLimit) { + const limit = specifiedLimit || config.media.limit; + + if (photos.length <= limit) { + return photos; + } + + const plucked = [1] + .concat( + Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (photos.length / (limit - 1)))), + ); + + return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close +} + +async function getEntropy(buffer) { + try { + const { entropy } = await sharp(buffer).stats(); + + return entropy; + } catch (error) { + logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`); + + return 7.5; + } +} + +async function createThumbnail(buffer) { + try { + const thumbnail = sharp(buffer) + .resize({ + height: config.media.thumbnailSize, + withoutEnlargement: true, + }) + .jpeg({ + quality: config.media.thumbnailQuality, + }) + .toBuffer(); + + return thumbnail; + } catch (error) { + logger.error(`Failed to create thumbnail: ${error.message}`); + } + + return null; +} + +async function createMediaDirectory(domain, subpath) { + const filepath = path.join(config.media.path, domain, subpath); + + await fs.mkdir(filepath, { recursive: true }); + return filepath; +} + +function curatePhotoEntries(files) { + return files.map((file, index) => ({ + path: file.filepath, + thumbnail: file.thumbpath, + mime: file.mimetype, + hash: file.hash, + source: file.source, + index, + })); +} + +async function findDuplicates(photos, identifier, prop = null) { + const duplicates = await knex('media') + .whereIn(identifier, photos.flat().map((photo) => { + if (prop) return photo[prop]; + if (photo.src) return photo.src; + + return photo; + })); + + const duplicateLookup = new Set(duplicates.map(photo => photo[prop || identifier])); + + const originals = photos.filter((source) => { + if (Array.isArray(source)) { + return !source.some(sourceX => duplicateLookup.has((prop && sourceX[prop]) || (sourceX.src && sourceX))); + } + + return !duplicateLookup.has((prop && source[prop]) || (source.src && source)); + }); + + return [duplicates, originals]; +} + +async function extractPhoto(source) { + const res = await bhttp.get(source.src); + + if (res.statusCode === 200) { + const { q } = ex(res.body.toString()); + + return source.extract(q); + } + + return null; +} + +async function fetchPhoto(photoUrl, index, label, attempt = 1) { + if (photoUrl.src && photoUrl.extract) { + // source links to page containing a (presumably) tokenized photo + const photo = await extractPhoto(photoUrl); + + return fetchPhoto(photo, index, label); + } + + if (Array.isArray(photoUrl)) { + return photoUrl.reduce(async (outcome, url) => outcome.catch(async () => { + const photo = await fetchPhoto(url, index, label); + + if (photo) { + return photo; + } + + throw new Error('Photo not available'); + }), Promise.reject(new Error())); + } + + try { + const { pathname } = new URL(photoUrl); + const res = await bhttp.get(photoUrl); + + if (res.statusCode === 200) { + const mimetype = mime.getType(pathname); + const extension = mime.getExtension(mimetype); + const hash = getHash(res.body); + const entropy = await getEntropy(res.body); + + return { + photo: res.body, + mimetype, + extension, + hash, + entropy, + source: photoUrl, + }; + } + + throw new Error(`Response ${res.statusCode} not OK`); + } catch (error) { + logger.warn(`Failed attempt ${attempt}/3 to fetch photo ${index + 1} for ${label} (${photoUrl}): ${error}`); + + if (attempt < 3) { + await Promise.delay(5000); + return fetchPhoto(photoUrl, index, label, attempt + 1); + } + + return null; + } +} + +async function savePhotos(files, { + domain = 'release', + subpath, + role = 'photo', + naming = 'index', +}) { + return Promise.map(files, async (file, index) => { + try { + const timestamp = new Date().getTime(); + const thumbnail = await createThumbnail(file.photo); + + const filename = naming === 'index' + ? `${file.role || role}${index + 1}` + : `${timestamp + index}`; + + const filepath = path.join(`${domain}s`, subpath, `${filename}.${file.extension}`); + const thumbpath = path.join(`${domain}s`, subpath, `${filename}_thumb.${file.extension}`); + + await Promise.all([ + fs.writeFile(path.join(config.media.path, filepath), file.photo), + fs.writeFile(path.join(config.media.path, thumbpath), thumbnail), + ]); + + return { + ...file, + thumbnail, + filepath, + thumbpath, + }; + } catch (error) { + logger.error(`Failed to store ${domain} ${role} to ${subpath}: ${error.message}`); + return null; + } + }); +} + +async function storePhotos(photos, { + domain = 'release', + role = 'photo', + naming = 'index', + targetId, + subpath, + primaryRole, // role to assign to first photo if not already in database, used mainly for avatars + entropyFilter = 2.5, // filter out fallback avatars and other generic clipart +}, label) { + if (!photos || photos.length === 0) { + logger.info(`No ${role}s available for ${label}`); + return; + } + + const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration + const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label); + + logger.info(`Fetching ${sourceOriginals.length} new ${role}s, ${sourceDuplicates.length} already present by source for ${label}`); + + const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), { + concurrency: 10, + }).filter(photo => photo && photo.entropy > entropyFilter); + + const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks + const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label); + + logger.info(`Saving ${hashOriginals.length} new ${role}s, ${hashDuplicates.length} already present by hash for ${label}`); + + const savedPhotos = await savePhotos(hashOriginals, { + domain, + role, + targetId, + subpath, + naming, + }); + + const curatedPhotoEntries = curatePhotoEntries(savedPhotos.filter(Boolean), domain, role, targetId); + + const newPhotos = await knex('media').insert(curatedPhotoEntries).returning('*'); + const photoEntries = Array.isArray(newPhotos) + ? [...sourceDuplicates, ...hashDuplicates, ...newPhotos] + : [...sourceDuplicates, ...hashDuplicates]; + + const photoAssociations = photoEntries + .map(photoEntry => ({ + [`${domain}_id`]: targetId, + media_id: photoEntry.id, + })); + + if (primaryRole) { + // store one photo as a 'primary' photo, such as an avatar or cover + const primaryPhoto = await knex(`${domain}s_${primaryRole}s`) + .where(`${domain}_id`, targetId) + .first(); + + if (primaryPhoto) { + const remainingAssociations = photoAssociations.filter(association => association.media_id !== primaryPhoto.media_id); + + await upsert(`${domain}s_${role}s`, remainingAssociations, [`${domain}_id`, 'media_id']); + return; + } + + await Promise.all([ + upsert(`${domain}s_${primaryRole}s`, photoAssociations.slice(0, 1), [`${domain}_id`, 'media_id']), + upsert(`${domain}s_${role}s`, photoAssociations.slice(1), [`${domain}_id`, 'media_id']), + ]); + + return; + } + + await upsert(`${domain}s_${role}s`, photoAssociations, [`${domain}_id`, 'media_id']); +} + +/* +async function storeReleasePhotos(releases, label) { + const sources = releases.map(release => pluckPhotos(release.photos)).flat(); + const uniqueSources = Array.from(new Set(sources)); + + const [sourceDuplicates, sourceOriginals] = await findDuplicates(uniqueSources, 'source', null, label); + + const metaFiles = await Promise.map( + sourceOriginals, + async (photoUrl, index) => fetchPhoto(photoUrl, index, label), + { concurrency: 10 }, + ) + .filter(photo => photo); + + const hashUniques = Object.values(metaFiles.reduce((acc, file) => { + if (!acc[file.hash]) acc[file.hash] = file; + + return acc; + }, {})); + + const [hashDuplicates, hashOriginals] = await findDuplicates(hashUniques, 'hash', 'hash', label); + + const sourceHashes = metaFiles.concat(sourceDuplicates).reduce((acc, file) => { + acc[file.source] = file.hash; + return acc; + }, {}); + + const associations = releases.map(release => release.photos.map(source => [release.id, sourceHashes[source]])).flat(); + + console.log(associations); +} +*/ + +async function storeTrailer(trailers, { + domain = 'releases', + role = 'trailer', + targetId, + subpath, +}, label) { + // support scrapers supplying multiple qualities + const trailer = Array.isArray(trailers) + ? trailers.find(trailerX => config.media.trailerQuality.includes(trailerX.quality)) || trailers[0] + : trailers; + + if (!trailer || !trailer.src) { + logger.info(`No ${role} available for ${label}`); + return; + } + + const [sourceDuplicates, sourceOriginals] = await findDuplicates([trailer], 'source', 'src', label); + + const metaFiles = await Promise.map(sourceOriginals, async (trailerX) => { + const { pathname } = new URL(trailerX.src); + const mimetype = trailerX.type || mime.getType(pathname); + + const res = await bhttp.get(trailerX.src); + const hash = getHash(res.body); + const filepath = path.join(domain, subpath, `${role}${trailerX.quality ? `_${trailerX.quality}` : ''}.${mime.getExtension(mimetype)}`); + + return { + trailer: res.body, + path: filepath, + mime: mimetype, + source: trailerX.src, + quality: trailerX.quality || null, + hash, + }; + }); + + const [hashDuplicates, hashOriginals] = await findDuplicates(metaFiles, 'hash', 'hash', label); + + const newTrailers = await knex('media') + .insert(hashOriginals.map(trailerX => ({ + path: trailerX.path, + mime: trailerX.mime, + source: trailerX.source, + quality: trailerX.quality, + hash: trailerX.hash, + type: role, + }))) + .returning('*'); + + await Promise.all(hashOriginals.map(trailerX => fs.writeFile(path.join(config.media.path, trailerX.path), trailerX.trailer))); + + const trailerEntries = Array.isArray(newTrailers) + ? [...sourceDuplicates, ...hashDuplicates, ...newTrailers] + : [...sourceDuplicates, ...hashDuplicates]; + + await upsert(`releases_${role}s`, trailerEntries.map(trailerEntry => ({ + release_id: targetId, + media_id: trailerEntry.id, + })), ['release_id', 'media_id']); +} + +module.exports = { + createMediaDirectory, + storePhotos, + // storeReleasePhotos, + storeTrailer, +}; diff --git a/src/releases.js b/src/releases.js index 638062fa..c7253115 100644 --- a/src/releases.js +++ b/src/releases.js @@ -10,12 +10,20 @@ const argv = require('./argv'); const whereOr = require('./utils/where-or'); const { associateTags } = require('./tags'); const { associateActors, scrapeBasicActors } = require('./actors'); +/* const { createMediaDirectory, storePhotos, // storeReleasePhotos, storeTrailer, + storeReleaseMedia, } = require('./media'); +*/ +const { + createMediaDirectory, + storePhotos, + storeTrailer, +} = require('./media_legacy'); const { fetchSites, findSiteByUrl } = require('./sites'); const slugify = require('./utils/slugify'); @@ -331,6 +339,8 @@ function accumulateMovies(releases) { async function storeReleaseAssets(releases) { // await storeReleasePhotos(releases); + // return storeReleaseMedia(releases); + await Promise.map(releases, async (release) => { const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`; const identifier = `"${release.title}" (${release.id})`; diff --git a/src/scrapers/pornpros.js b/src/scrapers/pornpros.js index afbcb2fe..3f9bdb07 100644 --- a/src/scrapers/pornpros.js +++ b/src/scrapers/pornpros.js @@ -8,7 +8,7 @@ function scrapeLatest(html, site) { const { document } = new JSDOM(html).window; const { origin } = new URL(site.url); - const videos = document.querySelectorAll('.video-releases-list').slice(-1)[0]; + const videos = Array.from(document.querySelectorAll('.video-releases-list')).slice(-1)[0]; return Array.from(videos.querySelectorAll('.card'), (scene) => { const release = { site }; diff --git a/src/utils/scorelogos.js b/src/utils/scorelogos.js new file mode 100644 index 00000000..45c555c9 --- /dev/null +++ b/src/utils/scorelogos.js @@ -0,0 +1,37 @@ +'use strict'; + +const Promise = require('bluebird'); +const bhttp = require('bhttp'); +const fs = require('fs-extra'); +const knex = require('../knex'); + +async function init() { + const sites = await knex('sites') + .select('networks.name', 'sites.slug') + .join('networks', 'networks.id', 'sites.network_id') + .where('networks.slug', 'score'); + + await Promise.map(sites, async (site) => { + const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`; + + console.log(url); + + const res = await bhttp.get(url, { + responseTimeout: 5000, + }); + + if (res.statusCode === 200) { + console.log(`Saving logo for ${site.slug}`); + + await fs.writeFile(`./score/${site.slug}.png`, res.body); + } + + console.log(`No logo found for ${site.slug}`); + }, { + concurrency: 10, + }); + + knex.destroy(); +} + +init();