From 5745cd33d8e8939490ada707ed55a6785e5b3b2e Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Mon, 4 Nov 2019 05:47:37 +0100 Subject: [PATCH] Added Dogfart scraper. Added 'date added' property to release page. --- assets/components/home/home.vue | 10 + assets/components/icon/icon.vue | 4 +- assets/components/release/release.vue | 24 +- assets/img/database-add.svg | 6 + assets/img/drawer-in.svg | 6 + migrations/20190325001339_releases.js | 2 +- package-lock.json | 391 ++++++++++++++++++++----- package.json | 1 + seeds/00_networks.js | 7 + seeds/01_sites.js | 162 ++++++++++ seeds/02_tags.js | 8 + seeds/{04_studios.js => 03_studios.js} | 0 src/argv.js | 5 + src/fetch-releases.js | 5 +- src/fetch-scene.js | 7 +- src/releases.js | 3 +- src/scrapers/21sextury.js | 2 +- src/scrapers/bangbros.js | 10 +- src/scrapers/dogfart.js | 170 +++++++++++ src/scrapers/index.js | 3 + src/scrapers/kink.js | 6 +- src/scrapers/mofos.js | 2 +- src/scrapers/naughtyamerica.js | 2 +- src/scrapers/realitykings.js | 6 + src/scrapers/xempire.js | 7 +- 25 files changed, 747 insertions(+), 102 deletions(-) create mode 100644 assets/img/database-add.svg create mode 100644 assets/img/drawer-in.svg rename seeds/{04_studios.js => 03_studios.js} (100%) create mode 100644 src/scrapers/dogfart.js diff --git a/assets/components/home/home.vue b/assets/components/home/home.vue index 0b6c893d..22e45e65 100644 --- a/assets/components/home/home.vue +++ b/assets/components/home/home.vue @@ -76,11 +76,21 @@ >{{ release.site.name }} {{ formatDate(release.date, 'MMM D, YYYY') }} + + {{ `(${formatDate(release.dateAdded, 'MMM D, YYYY')})` }}`
- + -
diff --git a/assets/img/database-add.svg b/assets/img/database-add.svg new file mode 100644 index 00000000..90af0adf --- /dev/null +++ b/assets/img/database-add.svg @@ -0,0 +1,6 @@ + + +database-add + + + diff --git a/assets/img/drawer-in.svg b/assets/img/drawer-in.svg new file mode 100644 index 00000000..77214a68 --- /dev/null +++ b/assets/img/drawer-in.svg @@ -0,0 +1,6 @@ + + +drawer-in + + + diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 7c063bff..32fa68b0 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -53,7 +53,7 @@ exports.up = knex => Promise.resolve() table.string('name'); table.string('url'); table.text('description'); - table.enum('orientation', ['gay', 'bi', 'lesbian', 'trans']); + table.string('parameters'); table.string('slug', 32) .unique(); diff --git a/package-lock.json b/package-lock.json index e57595ad..547bbb99 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1462,6 +1462,11 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "abab": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.2.tgz", + "integrity": "sha512-2scffjvioEmNz0OyDSLGWDfKCVwaKc6l9Pm9kOIREU13ClXZvHpg/nRL5xyjSSSLhOnXqft2HpsAzNEEA8cFFg==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1479,8 +1484,7 @@ "acorn": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", - "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==", - "dev": true + "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==" }, "acorn-dynamic-import": { "version": "4.0.0", @@ -1488,17 +1492,30 @@ "integrity": "sha512-d3OEjQV4ROpoflsnUA8HozoIR504TFxNivYEUi6uwz0IYhBkTDXGuWlNdMtybRt3nqVx/L6XqMt0FxkXuWKZhw==", "dev": true }, + "acorn-globals": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz", + "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==", + "requires": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, "acorn-jsx": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz", "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg==", "dev": true }, + "acorn-walk": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", + "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==" + }, "ajv": { "version": "6.9.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz", "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==", - "dev": true, "requires": { "fast-deep-equal": "^2.0.1", "fast-json-stable-stringify": "^2.0.0", @@ -1624,6 +1641,11 @@ "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz", "integrity": "sha1-p5SvDAWrF1KEbudTofIRoFugxE8=" }, + "array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, "array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -1659,7 +1681,6 @@ "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dev": true, "requires": { "safer-buffer": "~2.1.0" } @@ -1705,8 +1726,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "dev": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assign-symbols": { "version": "1.0.0", @@ -1737,11 +1757,15 @@ "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI=", "dev": true }, + "async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", - "dev": true + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "atob": { "version": "2.1.2", @@ -1765,14 +1789,12 @@ "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "dev": true + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==", - "dev": true + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "axobject-query": { "version": "2.0.2", @@ -2366,7 +2388,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dev": true, "requires": { "tweetnacl": "^0.14.3" } @@ -2568,6 +2589,11 @@ "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", "dev": true }, + "browser-process-hrtime": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz", + "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==" + }, "browserify-aes": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", @@ -2779,8 +2805,7 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", - "dev": true + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "chalk": { "version": "2.4.2", @@ -3000,7 +3025,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -3333,6 +3357,26 @@ "integrity": "sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==", "dev": true }, + "cssom": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.1.tgz", + "integrity": "sha512-6Aajq0XmukE7HdXUU6IoSWuH1H6gH9z6qmagsstTiN7cW2FNTsb+J2Chs+ufPgZCsV/yo8oaEudQLrb9dGxSVQ==" + }, + "cssstyle": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.0.0.tgz", + "integrity": "sha512-QXSAu2WBsSRXCPjvI43Y40m6fMevvyRm8JVAuF9ksQz5jha4pWP1wpaK7Yu5oLFc6+XAY+hj8YhefyXcBB53gg==", + "requires": { + "cssom": "~0.3.6" + }, + "dependencies": { + "cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + } + } + }, "csv": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/csv/-/csv-4.0.0.tgz", @@ -3387,11 +3431,20 @@ "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } }, + "data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "requires": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, "date-now": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", @@ -3430,8 +3483,7 @@ "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", - "dev": true + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" }, "defaults": { "version": "1.0.3", @@ -3490,8 +3542,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", - "dev": true + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "delegates": { "version": "1.0.0", @@ -3578,6 +3629,14 @@ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" }, + "domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "requires": { + "webidl-conversions": "^4.0.2" + } + }, "domhandler": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", @@ -3611,7 +3670,6 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "dev": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -3751,6 +3809,31 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "escodegen": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.12.0.tgz", + "integrity": "sha512-TuA+EhsanGcme5T3R0L80u4t8CpbXQjegRmf7+FPTJrtCTErXFeelblRgHQa1FofEzqYYJmJ/OqjTwREp9qgmg==", + "requires": { + "esprima": "^3.1.3", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + }, + "dependencies": { + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true + } + } + }, "eslint": { "version": "5.16.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", @@ -4080,8 +4163,7 @@ "estraverse": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", - "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", - "dev": true + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" }, "esutils": { "version": "2.0.2", @@ -4421,26 +4503,22 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "dev": true + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", - "dev": true + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "fast-json-stable-stringify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", - "dev": true + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", - "dev": true + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "figgy-pudding": { "version": "3.5.1", @@ -4656,14 +4734,12 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "dev": true + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -5428,7 +5504,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -5553,14 +5628,12 @@ "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "dev": true + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "dev": true, "requires": { "ajv": "^6.5.5", "har-schema": "^2.0.0" @@ -5705,6 +5778,14 @@ "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", "dev": true }, + "html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "requires": { + "whatwg-encoding": "^1.0.1" + } + }, "htmlparser2": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", @@ -5745,7 +5826,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -6252,8 +6332,7 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "dev": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "is-unc-path": { "version": "1.0.0", @@ -6297,8 +6376,7 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", - "dev": true + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "js-base64": { "version": "2.5.1", @@ -6341,8 +6419,52 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "dev": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsdom": { + "version": "15.2.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-15.2.0.tgz", + "integrity": "sha512-+hRyEfjRPFwTYMmSQ3/f7U9nP8ZNZmbkmUek760ZpxnCPWJIhaaLRuUSvpJ36fZKCGENxLwxClzwpOpnXNfChQ==", + "requires": { + "abab": "^2.0.0", + "acorn": "^7.1.0", + "acorn-globals": "^4.3.2", + "array-equal": "^1.0.0", + "cssom": "^0.4.1", + "cssstyle": "^2.0.0", + "data-urls": "^1.1.0", + "domexception": "^1.0.1", + "escodegen": "^1.11.1", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.1.4", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.7", + "saxes": "^3.1.9", + "symbol-tree": "^3.2.2", + "tough-cookie": "^3.0.1", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.1.2", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^7.0.0", + "xml-name-validator": "^3.0.0" + }, + "dependencies": { + "acorn": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", + "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==" + }, + "parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" + } + } }, "jsesc": { "version": "2.5.2", @@ -6358,14 +6480,12 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", - "dev": true + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -6376,8 +6496,7 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", - "dev": true + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "json5": { "version": "2.1.0", @@ -6406,7 +6525,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -6526,7 +6644,6 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "dev": true, "requires": { "prelude-ls": "~1.1.2", "type-check": "~0.3.2" @@ -6689,6 +6806,11 @@ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, "lodash.tail": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.tail/-/lodash.tail-4.1.1.tgz", @@ -7482,11 +7604,15 @@ "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, + "nwsapi": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.1.4.tgz", + "integrity": "sha512-iGfd9Y6SFdTNldEy2L0GUhcarIutFmk+MPWIn9dmj8NMIup03G08uUF2KGbbmv/Ux4RT0VZJoP/sVbWA6d/VIw==" + }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "dev": true + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "object-assign": { "version": "4.1.1", @@ -7665,7 +7791,6 @@ "version": "0.8.2", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", - "dev": true, "requires": { "deep-is": "~0.1.3", "fast-levenshtein": "~2.0.4", @@ -7968,8 +8093,7 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", - "dev": true + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "pg": { "version": "7.9.0", @@ -8070,6 +8194,11 @@ "find-up": "^2.1.0" } }, + "pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", @@ -8253,8 +8382,7 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" }, "prepend-http": { "version": "1.0.4", @@ -8751,7 +8879,6 @@ "version": "2.88.0", "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "dev": true, "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -8778,20 +8905,17 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "dev": true + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", - "dev": true + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "dev": true, "requires": { "psl": "^1.1.24", "punycode": "^1.4.1" @@ -8800,8 +8924,36 @@ "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "dev": true + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, + "request-promise-core": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz", + "integrity": "sha512-UHYyq1MO8GsefGEt7EprS8UrXsm1TxEvFUX1IMTuSLU2Rh7fTIdFtl8xD7JiEYiWU2dl+NYAjCTksTehQUxPag==", + "requires": { + "lodash": "^4.17.11" + } + }, + "request-promise-native": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.7.tgz", + "integrity": "sha512-rIMnbBdgNViL37nZ1b3L/VfPOpSi0TqVDQPAvO6U14lMzOLrt5nilxCQqtDKhZeDiW0/hkCXGoQjhgJd/tCh6w==", + "requires": { + "request-promise-core": "1.1.2", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } } } }, @@ -9190,6 +9342,14 @@ } } }, + "saxes": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", + "integrity": "sha512-Ydydq3zC+WYDJK1+gRxRapLIED9PWeSuuS41wqyoRmzvhhh9nc+QQrVMKJYzJFULazeGhzSV0QleN2wD3boh2g==", + "requires": { + "xmlchars": "^2.1.1" + } + }, "scheduler": { "version": "0.13.6", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.13.6.tgz", @@ -9682,7 +9842,6 @@ "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "dev": true, "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -9737,6 +9896,11 @@ "readable-stream": "^2.0.1" } }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, "stream-browserify": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", @@ -9884,6 +10048,11 @@ "has-flag": "^3.0.0" } }, + "symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" + }, "table": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/table/-/table-5.4.0.tgz", @@ -10131,6 +10300,14 @@ "punycode": "^2.1.1" } }, + "tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "requires": { + "punycode": "^2.1.0" + } + }, "trim-newlines": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", @@ -10271,7 +10448,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, "requires": { "safe-buffer": "^5.0.1" } @@ -10279,14 +10455,12 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "dev": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dev": true, "requires": { "prelude-ls": "~1.1.2" } @@ -10461,7 +10635,6 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "dev": true, "requires": { "punycode": "^2.1.0" } @@ -10557,7 +10730,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -10685,6 +10857,24 @@ "resolved": "https://registry.npmjs.org/vuex/-/vuex-3.1.1.tgz", "integrity": "sha512-ER5moSbLZuNSMBFnEBVGhQ1uCBNJslH9W/Dw2W7GZN23UQA69uapP5GTT9Vm8Trc0PzBSVt6LzF3hGjmv41xcg==" }, + "w3c-hr-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz", + "integrity": "sha1-gqwr/2PZUOqeMYmlimViX+3xkEU=", + "requires": { + "browser-process-hrtime": "^0.1.2" + } + }, + "w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "requires": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, "watchpack": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.6.0.tgz", @@ -10704,6 +10894,11 @@ "defaults": "^1.0.3" } }, + "webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, "webpack": { "version": "4.32.2", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.32.2.tgz", @@ -10872,6 +11067,29 @@ } } }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "requires": { + "iconv-lite": "0.4.24" + } + }, + "whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -10897,8 +11115,7 @@ "wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", - "dev": true + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" }, "worker-farm": { "version": "1.7.0", @@ -10965,6 +11182,24 @@ "mkdirp": "^0.5.1" } }, + "ws": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.0.tgz", + "integrity": "sha512-+SqNqFbwTm/0DC18KYzIsMTnEWpLwJsiasW/O17la4iDRRIO9uaHbvKiAS3AHgTiuuWerK/brj4O6MYZkei9xg==", + "requires": { + "async-limiter": "^1.0.0" + } + }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, "xtend": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xtend/-/xtend-3.0.0.tgz", diff --git a/package.json b/package.json index 67ed5b0f..4d33e35f 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "express-promise-router": "^3.0.3", "express-react-views": "^0.11.0", "fs-extra": "^7.0.1", + "jsdom": "^15.2.0", "knex": "^0.16.3", "knex-migrate": "^1.7.1", "mime": "^2.4.4", diff --git a/seeds/00_networks.js b/seeds/00_networks.js index ca950512..36398c57 100644 --- a/seeds/00_networks.js +++ b/seeds/00_networks.js @@ -31,6 +31,13 @@ exports.seed = knex => Promise.resolve() url: 'https://ddfnetwork.com', description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.', }, + { + slug: 'dogfartnetwork', + name: 'Dogfart Network', + url: 'https://dogfartnetwork.com', + description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.', + parameters: JSON.stringify({ photoLimit: 25 }), + }, { slug: 'evilangel', name: 'Evil Angel', diff --git a/seeds/01_sites.js b/seeds/01_sites.js index af0fb923..44f92698 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -828,6 +828,168 @@ exports.seed = knex => Promise.resolve() description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars', network_id: networksMap['ddfnetwork'], }, + // DOGFART NETWORK + { + slug: 'blacksonblondes', + name: 'Blacks On Blondes', + url: 'https://blacksonblondes.com/tour', + description: 'Blacks On Blondes is the Worlds Largest and Best Interracial Sex and Interracial Porn website. Black Men and White Women. BlacksOnBlondes has 23 years worth of Hardcore Interracial Content. Featuring the entire Legendary Dogfart Movie Archive', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'cuckoldsessions', + name: 'Cuckold Sessions', + url: 'https://cuckoldsessions.com/tour', + description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com/tour - Hardcore Cuckold Fetish Videos', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'gloryhole', + name: 'Glory Hole', + url: 'https://gloryhole.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'blacksoncougars', + name: 'Blacks On Cougars', + url: 'https://blacksoncougars.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'wefuckblackgirls', + name: 'We Fuck Black Girls', + url: 'https://wefuckblackgirls.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'watchingmymomgoblack', + name: 'Watching My Mom Go Black', + url: 'https://watchingmymomgoblack.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'interracialblowbang', + name: 'Interracial Blowbang', + url: 'https://interracialblowbang.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'cumbang', + name: 'Cumbang', + url: 'https://cumbang.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'interracialpickups', + name: 'Interracial Pickups', + url: 'https://interracialpickups.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'watchingmydaughtergoblack', + name: 'Watching My Daughter Go Black', + url: 'https://watchingmydaughtergoblack.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'zebragirls', + name: 'Zebra Girls', + url: 'https://zebragirls.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'gloryholeinitiations', + name: 'Gloryhole Initiations', + url: 'https://gloryhole-initiations.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'dogfartbehindthescenes', + name: 'Dogfart Behind The Scenes', + url: 'https://dogfartbehindthescenes.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'blackmeatwhitefeet', + name: 'Black Meat White Feet', + url: 'https://blackmeatwhitefeet.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'springthomas', + name: 'Spring Thomas', + url: 'https://springthomas.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'katiethomas', + name: 'Katie Thomas', + url: 'https://katiethomas.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'ruthblackwell', + name: 'Ruth Blackwell', + url: 'https://ruthblackwell.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'candymonroe', + name: 'Candy Monroe', + url: 'https://candymonroe.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'wifewriting', + name: 'Wife Writing', + url: 'https://wifewriting.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'barbcummings', + name: 'Barb Cummings', + url: 'https://barbcummings.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'theminion', + name: 'The Minion', + url: 'https://theminion.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'blacksonboys', + name: 'Blacks On Boys', + url: 'https://blacksonboys.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, + { + slug: 'gloryholesandhandjobs', + name: 'Gloryholes And Handjobs', + url: 'https://gloryholesandhandjobs.com/tour', + description: '', + network_id: networksMap['dogfartnetwork'], + }, // EVIL ANGEL { slug: 'evilangel', diff --git a/seeds/02_tags.js b/seeds/02_tags.js index 12b902b1..d23bf9df 100644 --- a/seeds/02_tags.js +++ b/seeds/02_tags.js @@ -805,6 +805,10 @@ exports.seed = knex => Promise.resolve() name: '2-on-1', alias_for: tagsMap['mfm'], }, + { + name: '2 on 1', + alias_for: tagsMap['mfm'], + }, { name: '3+ on 1', alias_for: tagsMap['gangbang'], @@ -873,6 +877,10 @@ exports.seed = knex => Promise.resolve() name: 'big dick', alias_for: tagsMap['big-cock'], }, + { + name: 'big booty', + alias_for: tagsMap['big-butt'], + }, { name: 'big butts', alias_for: tagsMap['big-butt'], diff --git a/seeds/04_studios.js b/seeds/03_studios.js similarity index 100% rename from seeds/04_studios.js rename to seeds/03_studios.js diff --git a/src/argv.js b/src/argv.js index 04001425..abed2329 100644 --- a/src/argv.js +++ b/src/argv.js @@ -30,6 +30,11 @@ const { argv } = yargs type: 'string', default: config.fetchAfter.join(' '), }) + .option('pages', { + describe: 'Limit pages to scrape per site. Only used when no dates are found or --after is unset.', + type: 'number', + default: 1, + }) .option('save', { describe: 'Save fetched releases to database', type: 'boolean', diff --git a/src/fetch-releases.js b/src/fetch-releases.js index 3230a987..4bc1356c 100644 --- a/src/fetch-releases.js +++ b/src/fetch-releases.js @@ -44,6 +44,7 @@ function curateSites(sites) { id: site.network_id, name: site.network_name, slug: site.network_slug, + parameters: JSON.parse(site.network_parameters), }, parameters: JSON.parse(site.parameters), })); @@ -55,7 +56,7 @@ async function accumulateIncludedSites() { const networkIds = networks.map(network => network.id); const rawSites = await knex('sites') - .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug') + .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.parameters as network_parameters') .whereIn('sites.slug', argv.sites || []) .orWhereIn('network_id', networkIds) .leftJoin('networks', 'sites.network_id', 'networks.id'); @@ -269,7 +270,7 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page const oldestReleaseOnPage = latestReleases.slice(-1)[0].date; - if (uniqueReleases.length > 0 && moment(oldestReleaseOnPage).isAfter(afterDate)) { + if (uniqueReleases.length > 0 && moment(oldestReleaseOnPage).isAfter(afterDate) && (!oldestReleaseOnPage && page < argv.pages)) { return fetchNewReleases(scraper, site, afterDate, accReleases.concat(uniqueReleases), page + 1); } diff --git a/src/fetch-scene.js b/src/fetch-scene.js index b611ce5a..0af6bae3 100644 --- a/src/fetch-scene.js +++ b/src/fetch-scene.js @@ -12,7 +12,7 @@ async function findSite(url) { const domain = hostname.replace(/^www./, ''); const site = await knex('sites') - .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug') + .select('sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.parameters as network_parameters') .where('sites.url', 'like', `%${domain}`) .leftJoin('networks', 'sites.network_id', 'networks.id') .first() @@ -30,6 +30,7 @@ async function findSite(url) { network: { id: site.network_id || site.id, slug: site.network_slug || site.slug, + parameters: site.network_parameters && JSON.parse(site.network_parameters), }, parameters: site.parameters && JSON.parse(site.parameters), isFallback: site.network_id === undefined, @@ -94,8 +95,8 @@ async function storeRelease(release) { return release; } -async function fetchScene(url) { - const site = await findSite(url); +async function fetchScene(url, release) { + const site = release.site || await findSite(url); const scraper = scrapers[site.slug] || scrapers[site.network.slug]; if (!scraper) { diff --git a/src/releases.js b/src/releases.js index 5f844b0b..7f80f4ee 100644 --- a/src/releases.js +++ b/src/releases.js @@ -21,6 +21,7 @@ async function curateRelease(release) { id: release.id, title: release.title, date: release.date, + dateAdded: release.created_at, description: release.description, url: release.url, shootId: release.shoot_id, @@ -73,7 +74,7 @@ async function fetchReleases(releaseId) { .leftJoin('sites', 'releases.site_id', 'sites.id') .leftJoin('studios', 'releases.studio_id', 'studios.id') .leftJoin('networks', 'sites.network_id', 'networks.id') - .orderBy('date', 'desc') + .orderBy([{ column: 'date', order: 'desc' }, { column: 'created_at', order: 'desc' }]) .limit(100); return curateReleases(releases); diff --git a/src/scrapers/21sextury.js b/src/scrapers/21sextury.js index 4fbd389b..d457a5af 100644 --- a/src/scrapers/21sextury.js +++ b/src/scrapers/21sextury.js @@ -91,7 +91,7 @@ async function scrapeScene(html, url, site) { const [channelSite, tags] = await Promise.all([ site.isFallback ? knex('sites') - .where({ id: siteId }) + .where({ slug: siteId }) .orWhereRaw('name = ? collate NOCASE', [siteName]) .first() : site, diff --git a/src/scrapers/bangbros.js b/src/scrapers/bangbros.js index 5d3bfe6a..7a0f88bf 100644 --- a/src/scrapers/bangbros.js +++ b/src/scrapers/bangbros.js @@ -69,10 +69,12 @@ async function scrapeScene(html, url, site) { const rawTags = $('.vdoTags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const [channelSite, tags] = await Promise.all([ - knex('sites') - .where({ slug: siteId }) - .orWhere({ name: siteName }) - .first(), + site.isFallback + ? knex('sites') + .where({ slug: siteId }) + .orWhere({ name: siteName }) + .first() + : site, matchTags(rawTags), ]); diff --git a/src/scrapers/dogfart.js b/src/scrapers/dogfart.js new file mode 100644 index 00000000..c21ccbb1 --- /dev/null +++ b/src/scrapers/dogfart.js @@ -0,0 +1,170 @@ +'use strict'; + +/* eslint-disable newline-per-chained-call */ +const Promise = require('bluebird'); +const bhttp = require('bhttp'); +const { JSDOM } = require('jsdom'); +const moment = require('moment'); +const knex = require('knex'); + +const { matchTags } = require('../tags'); + +async function getPhoto(url) { + const res = await bhttp.get(url); + const html = res.body.toString(); + const { document } = new JSDOM(html).window; + + const photoUrl = document.querySelector('.scenes-module img').src; + + return photoUrl; +} + +async function getPhotos(albumUrl, site, siteUrl) { + const res = await bhttp.get(albumUrl); + const html = res.body.toString(); + const { document } = new JSDOM(html).window; + + const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href; + const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10); + + // dogfart has massive albums, pick 20 or specified photos: first, last and evenly inbetween + const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25; + const photoIndexes = [1] + .concat(Array.from({ length: photoLimit - 2 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 2))))) + .concat(lastPhotoIndex); + + if (photoLimit > 25) { + console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`); + } + + const photoUrls = await Promise.map(photoIndexes, async (index) => { + const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`; + + return getPhoto(pageUrl); + }, { + concurrency: 5, + }); + + return photoUrls; +} + +function scrapeLatest(html, site) { + const { document } = new JSDOM(html).window; + const sceneElements = Array.from(document.querySelectorAll('.recent-updates')); + + return sceneElements.reduce((acc, element) => { + const siteUrl = element.querySelector('.help-block').textContent; + + if (siteUrl.toLowerCase() !== new URL(site.url).host) { + // different dogfart site + return acc; + } + + const sceneLinkElement = element.querySelector('.thumbnail'); + const url = `https://dogfartnetwork.com${sceneLinkElement.href}`; + const { pathname } = new URL(url); + const entryId = `${site.slug}_${pathname.split('/')[4]}`; + + const title = element.querySelector('.scene-title').textContent; + const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim()); + + const poster = `https:${element.querySelector('img').src}`; + const trailer = sceneLinkElement.dataset.preview_clip_url; + + return [ + ...acc, + { + url, + entryId, + title, + actors, + poster, + trailer: { + src: trailer, + }, + site, + }, + ]; + }, []); +} + +async function scrapeScene(html, url, site) { + const { document } = new JSDOM(html).window; + + const title = document.querySelector('.description-title').textContent; + const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent); + const metaDescription = document.querySelector('meta[itemprop="description"]').content; + const description = metaDescription + ? metaDescription.content + : document.querySelector('.description') + .textContent + .replace(/[ \t\n]{2,}/g, ' ') + .replace('...read more', '') + .trim(); + + const siteSlug = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase(); + const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content); + const duration = moment + .duration(document + .querySelectorAll('.extra-info p')[1] + .textContent + .match(/\d+:\d+$/)[0]) + .asSeconds(); + + const trailerElement = document.querySelector('.html5-video'); + const poster = `https:${trailerElement.dataset.poster}`; + const { trailer } = trailerElement.dataset; + + const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href; + const { origin, pathname } = new URL(url); + const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url); + + const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2; + const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent); + + const [channelSite, tags] = await Promise.all([ + site.isFallback + ? knex('sites') + .where({ slug: siteSlug }) + .orWhere({ url: `https://${siteSlug}.com` }) + .first() + : site, + matchTags(rawTags), + ]); + + return { + url, + title, + description, + actors, + date, + duration, + poster, + photos, + trailer: { + src: trailer, + }, + tags, + rating: { + stars, + }, + site: channelSite || site, + }; +} + +async function fetchLatest(site, page = 1) { + const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`); + + return scrapeLatest(res.body.toString(), site); +} + +async function fetchScene(url, site) { + const res = await bhttp.get(url); + + return scrapeScene(res.body.toString(), url, site); +} + +module.exports = { + fetchLatest, + fetchScene, +}; diff --git a/src/scrapers/index.js b/src/scrapers/index.js index 3ca6637b..f0400a1d 100644 --- a/src/scrapers/index.js +++ b/src/scrapers/index.js @@ -5,6 +5,7 @@ const bangbros = require('./bangbros'); const blowpass = require('./blowpass'); const brazzers = require('./brazzers'); const ddfnetwork = require('./ddfnetwork'); +const dogfart = require('./dogfart'); const evilangel = require('./evilangel'); const julesjordan = require('./julesjordan'); const kink = require('./kink'); @@ -23,6 +24,8 @@ module.exports = { blowpass, brazzers, ddfnetwork, + dogfart, + dogfartnetwork: dogfart, evilangel, julesjordan, kink, diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js index 94eadcdb..111469c0 100644 --- a/src/scrapers/kink.js +++ b/src/scrapers/kink.js @@ -3,8 +3,8 @@ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); +const knex = require('knex'); -const knex = require('../knex'); const { matchTags } = require('../tags'); function scrapeLatest(html, site) { @@ -75,7 +75,9 @@ async function scrapeScene(html, url, shootId, ratingRes, site) { const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const [channelSite, tags] = await Promise.all([ - knex('sites').where({ slug: sitename }).first(), + site.isFallback + ? knex('sites').where({ slug: sitename }).first() + : site, matchTags(rawTags), ]); diff --git a/src/scrapers/mofos.js b/src/scrapers/mofos.js index 7325a942..a41360f7 100644 --- a/src/scrapers/mofos.js +++ b/src/scrapers/mofos.js @@ -58,7 +58,7 @@ async function scrapeScene(html, url, site) { const [channelSite, tags] = await Promise.all([ knex('sites') - .where({ id: siteId }) + .where({ slug: siteId }) .orWhere({ url: `https://www.mofos.com${siteUrl}` }) .orWhere({ name: sitename }) .first(), diff --git a/src/scrapers/naughtyamerica.js b/src/scrapers/naughtyamerica.js index c0e94c58..a055472c 100644 --- a/src/scrapers/naughtyamerica.js +++ b/src/scrapers/naughtyamerica.js @@ -70,7 +70,7 @@ async function scrapeScene(html, url, site) { const [channelSite, tags] = await Promise.all([ knex('sites') - .where({ id: siteId }) + .where({ slug: siteId }) .orWhere({ name: siteName }) .first(), matchTags(rawTags), diff --git a/src/scrapers/realitykings.js b/src/scrapers/realitykings.js index 72b740ca..faa84257 100644 --- a/src/scrapers/realitykings.js +++ b/src/scrapers/realitykings.js @@ -13,6 +13,8 @@ function scrapeLatest(html, site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const sceneElements = $('.card.card--release').toArray(); + console.log(sceneElements); + return sceneElements.map((element) => { const sceneLinkElement = $(element).find('.card-info__title a'); const title = sceneLinkElement.attr('title'); @@ -22,6 +24,8 @@ function scrapeLatest(html, site) { const date = moment.utc($(element).find('.card-info__meta-date').text(), 'MMMM DD, YYYY').toDate(); const actors = $(element).find('.card-info__cast a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray(); + console.log(date, actors, title); + return { url, entryId, @@ -54,6 +58,8 @@ async function scrapeScene(data, url, site) { const { likes, dislikes } = data.stats; const duration = data.videos.mediabook.length; + console.log(data); + const rawTags = data.tags.map(tag => tag.name); const tags = await matchTags(rawTags); diff --git a/src/scrapers/xempire.js b/src/scrapers/xempire.js index 99e8d22b..692c775a 100644 --- a/src/scrapers/xempire.js +++ b/src/scrapers/xempire.js @@ -3,9 +3,9 @@ const Promise = require('bluebird'); const bhttp = require('bhttp'); const cheerio = require('cheerio'); +const knex = require('knex'); const moment = require('moment'); -const knex = require('../knex'); const { matchTags } = require('../tags'); async function fetchPhotos(url) { @@ -126,7 +126,6 @@ async function scrapeScene(html, url, site) { const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds(); - const rawTags = data.keywords.split(', '); const siteDomain = $('meta[name="twitter:domain"]').attr('content'); const siteId = siteDomain && siteDomain.split('.')[0].toLowerCase(); const siteUrl = siteDomain && `https://www.${siteDomain}`; @@ -136,11 +135,13 @@ async function scrapeScene(html, url, site) { const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain); + const rawTags = data.keywords.split(', '); + const [channelSite, tags] = await Promise.all([ site.isFallback ? knex('sites') .where({ url: siteUrl }) - .orWhere({ id: siteId }) + .orWhere({ slug: siteId }) .first() : site, matchTags(rawTags),