Added scene scraping to Teen Core Club.

]

@@ -1,146 +0,0 @@
-<template>
-	<div
-		v-if="tag"
-		class="content"
-	>
-		<div
-			class="tag"
-			:class="{ nomedia: !hasMedia }"
-		>
-			<div class="header">
-				<h2 class="title">
-					<Icon icon="price-tag4" />
-					{{ tag.name }}
-				</h2>
-
-				<p
-					v-if="description"
-					class="description header-description"
-					v-html="description"
-				/>
-			</div>
-
-
-			<div class="content-inner">
-				<Photos
-					v-if="hasMedia"
-					:tag="tag"
-					class="compact"
-				/>
-
-				<FilterBar :fetch-releases="fetchReleases" />
-				<Releases :releases="tag.releases" />
-			</div>
-		</div>
-	</div>
-</template>
-
-<script>
-/* eslint-disable no-v-html */
-import { Converter } from 'showdown';
-
-import escapeHtml from '../../../src/utils/escape-html';
-
-import FilterBar from '../header/filter-bar.vue';
-import Photos from './photos.vue';
-import Releases from '../releases/releases.vue';
-
-const converter = new Converter();
-
-async function fetchReleases() {
-	this.tag = await this.$store.dispatch('fetchTagBySlug', {
-		tagSlug: this.$route.params.tagSlug,
-		range: this.$route.params.range,
-	});
-
-	this.hasMedia = this.tag.poster || this.tag.photos.length > 0;
-	this.description = this.tag.description && converter.makeHtml(escapeHtml(this.tag.description));
-}
-
-async function route() {
-	await this.fetchReleases();
-}
-
-async function mounted() {
-	await this.fetchReleases();
-	this.pageTitle = this.tag.name;
-}
-
-export default {
-	components: {
-		FilterBar,
-		Photos,
-		Releases,
-	},
-	data() {
-		return {
-			tag: null,
-			description: null,
-			releases: null,
-			pageTitle: null,
-			hasMedia: false,
-		};
-	},
-	watch: {
-		$route: route,
-	},
-	mounted,
-	methods: {
-		fetchReleases,
-	},
-};
-</script>
-
-<style lang="scss" scoped>
-@import 'theme';
-
-.tag {
-    flex-grow: 1;
-    overflow: hidden;
-
-    &.nomedia {
-        flex-direction: column;
-
-        .sidebar {
-            display: none;
-        }
-
-        .header {
-            display: flex;
-        }
-    }
-}
-
-.header {
-    background: var(--profile);
-    color: var(--text-light);
-    justify-content: space-between;
-    padding: .5rem 1rem;
-
-    .title {
-        margin: 0 2rem 0 0;
-    }
-}
-
-.title {
-    padding: 0;
-    margin: 0;
-    flex-shrink: 0;
-    text-transform: capitalize;
-
-    .icon {
-        fill: var(--text-light);
-        width: 1.25rem;
-        height: 1.25rem;
-    }
-}
-
-.description {
-    margin: 0;
-    line-height: 1.5;
-}
-
-.dark .sidebar {
-    border-right: solid 1px var(--shadow-hint);
-}
-</style>

package-lock.json

@@ -6906,6 +6906,15 @@
                 }
             }
         },
+        "m3u8stream": {
+            "version": "0.7.1",
+            "resolved": "https://registry.npmjs.org/m3u8stream/-/m3u8stream-0.7.1.tgz",
+            "integrity": "sha512-z6ldnAdhbuWOL6LmMkwptSZGzj+qbRytMKLTbNicwF/bJMjf9U9lqD57RNQUFecvWadEkzy6PDjcNJFFgi19uQ==",
+            "requires": {
+                "miniget": "^1.6.1",
+                "sax": "^1.2.4"
+            }
+        },
         "make-dir": {
             "version": "2.1.0",
             "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -7210,6 +7219,11 @@
                 "webpack-sources": "^1.1.0"
             }
         },
+        "miniget": {
+            "version": "1.7.2",
+            "resolved": "https://registry.npmjs.org/miniget/-/miniget-1.7.2.tgz",
+            "integrity": "sha512-USPNNK2bnHLOplX8BZVMehUkyQizS/DFpBdoH0TS+fM+hQoLNg9tWg4MeY9wE8gfY0pbzmx5UBEODujt3Lz8AA=="
+        },
         "minimalistic-assert": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",

package.json

@@ -99,6 +99,7 @@
         "knex": "^0.20.10",
         "knex-migrate": "^1.7.4",
         "longjohn": "^0.2.12",
+        "m3u8stream": "^0.7.1",
         "mime": "^2.4.4",
         "moment": "^2.24.0",
         "nanoid": "^2.1.11",

src/scrapers/teencoreclub.js

@@ -1,8 +1,12 @@
 'use strict';
 
+const fs = require('fs');
 const moment = require('moment');
+const m3u8stream = require('m3u8stream');
 
+const logger = require('../logger')(__filename);
 const http = require('../utils/http');
+const qu = require('../utils/qu');
 const { prefixUrl } = require('../utils/qu');
 
 function scrapeLatest(scenes, entity) {
@@ -36,12 +40,56 @@ function scrapeLatest(scenes, entity) {
 			release.tags = ['gay'];
 		}
 
-		console.log(scene);
-		console.log(release);
 		return release;
 	});
 }
 
+async function scrapeScene({ query }, url, entity) {
+	const release = {};
+
+	const entryId = new URL(url).pathname.match(/\/video\/(\d+)/)[1];
+	release.entryId = entryId;
+
+	const title = query.meta('name=title');
+
+	if (/bic/i.test(title)) {
+		release.shootId = title.toUpperCase().replace('-', '_');
+	} else {
+		release.title = title;
+	}
+
+	release.date = query.date('.detail-meta li:nth-child(2)', 'YYYY');
+	release.datePrecision = 'year';
+
+	release.description = query.q('.detail-description', true);
+	release.duration = query.dur('.detail-meta li:first-child');
+
+	release.actors = [query.q('.detail-hero-title h1', true)];
+
+	release.poster = query.q('.detail-hero').style['background-image'].match(/url\((.+)\)/)[1];
+	release.photos = query.imgs('.detail-grabs img');
+
+	const streamData = await http.get(`${entity.url}/video/source/${entryId}`, {
+		host: new URL(entity.url).host,
+		referer: url,
+	});
+
+	if (streamData.ok && streamData.body.status === 'success') {
+		console.log(streamData.body);
+
+		await m3u8stream(streamData.body.link)
+			.pipe(fs.createWriteStream('./trailer.mp4'))
+			.on('progress', status => console.log(status))
+			.on('error', error => console.log(error));
+	}
+
+	if (streamData.body.status !== 'success') {
+		logger.warn(`Could not retrieve trailer from ${entity.name} (Teen Core Club): ${streamData.body.status}`);
+	}
+
+	return release;
+}
+
 async function fetchLatest(entity, page = 1) {
 	// console.log(entity, page);
 
@@ -58,6 +106,17 @@ async function fetchLatest(entity, page = 1) {
 	return null;
 }
 
+async function fetchScene(url, entity) {
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, entity);
+	}
+
+	return res.status;
+}
+
 module.exports = {
 	fetchLatest,
+	fetchScene,
 };

src/utils/qu.js

@@ -5,7 +5,10 @@ const moment = require('moment');
 const http = require('./http');
 
 function trim(str) {
-	if (!str) return null;
+	if (typeof str !== 'string') {
+		return null;
+	}
+
 	return str.trim().replace(/\s+/g, ' ');
 }
 
@@ -263,6 +266,7 @@ function init(element, window) {
 		}),
 		...legacyContextFuncs,
 		qu: quContextFuncs,
+		query: quContextFuncs,
 	};
 }